From c132bce1a18480b3d545a056b8573b2f9b194af0 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Thu, 3 Feb 2022 08:31:05 -0700 Subject: [PATCH] MDEV-20119: Implement the --do-domain-ids, --ignore-domain-ids, and --ignore-server-ids options for mysqlbinlog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Feature: ============ Extend mariadb-binlog command-line tool to allow for filtering events using GTID domain and server ids. The functionality mimics that of a replica server’s DO_DOMAIN_IDS, IGNORE_DOMAIN_IDS, and IGNORE_SERVER_IDS from CHANGE MASTER TO. For completeness, this patch additionally adds the option --do-server-ids as an alias for --server-id, which now accepts a list of server ids instead of a single one. Example usage: mariadb-binlog --do-domain-ids=2,3,4 --do-server-ids=1,3 master-bin.000001 Functional Notes: 1. --do-domain-ids cannot be combined with --ignore-domain-ids 2. --do-server-ids cannot be combined with --ignore-server-ids 3. A domain id filter can be combined with a server id filter 4. When any new filter options are combined with the --gtid-strict-mode option, events from excluded domains/servers are not validated. 5. Domain/server id filters can be combined with GTID ranges (i.e. specifications of --start-position and --stop-position). However, because the --stop-position option implicitly undertakes filtering to only output events within its range of domains, when combined with --do-domain-ids or --ignore-domain-ids, output will consist of the intersection between the filters. Specifically, with --do-domain-ids and --stop-position, only events with domain ids present in both argument lists will be output. Conversely, with --ignore-domain-ids and --stop-position, only events with domain ids present in the --stop-position and absent from the --ignore-domain-ids options will be output. Reviewed By ============ Andrei Elkin --- client/client_priv.h | 4 + client/mysqlbinlog.cc | 469 ++++-- man/mysqlbinlog.1 | 67 + .../r/binlog_mysqlbinlog_do_server_ids.result | 161 ++ .../t/binlog_mysqlbinlog_do_server_ids.test | 477 ++++++ .../include/mysqlbinlog_slave_consistency.inc | 201 +++ .../include/rpl_check_table_consistency.inc | 63 + .../suite/rpl/include/sql_multisource.inc | 45 + .../rpl/include/sql_out_of_order_gtid.inc | 43 + .../rpl_mysqlbinlog_slave_consistency.result | 1371 +++++++++++++++++ .../t/rpl_mysqlbinlog_slave_consistency.test | 401 +++++ sql/rpl_gtid.cc | 269 +++- sql/rpl_gtid.h | 132 +- 13 files changed, 3521 insertions(+), 182 deletions(-) create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_do_server_ids.result create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_do_server_ids.test create mode 100644 mysql-test/suite/rpl/include/mysqlbinlog_slave_consistency.inc create mode 100644 mysql-test/suite/rpl/include/rpl_check_table_consistency.inc create mode 100644 mysql-test/suite/rpl/include/sql_multisource.inc create mode 100644 mysql-test/suite/rpl/include/sql_out_of_order_gtid.inc create mode 100644 mysql-test/suite/rpl/r/rpl_mysqlbinlog_slave_consistency.result create mode 100644 mysql-test/suite/rpl/t/rpl_mysqlbinlog_slave_consistency.test diff --git a/client/client_priv.h b/client/client_priv.h index 039b008c883..da9763aa252 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -106,6 +106,10 @@ enum options_client OPT_COPY_S3_TABLES, OPT_PRINT_TABLE_METADATA, OPT_ASOF_TIMESTAMP, + OPT_IGNORE_DOMAIN_IDS, + OPT_DO_DOMAIN_IDS, + OPT_IGNORE_SERVER_IDS, + OPT_DO_SERVER_IDS, OPT_MAX_CLIENT_OPTION /* should be always the last */ }; diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 21980ca03d6..173744204e1 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -47,6 +47,7 @@ #include "sql_common.h" #include "my_dir.h" #include // ORACLE_WELCOME_COPYRIGHT_NOTICE +#include "rpl_gtid.h" #include "sql_string.h" // needed for Rpl_filter #include "sql_list.h" // needed for Rpl_filter #include "rpl_filter.h" @@ -82,7 +83,7 @@ extern "C" { char server_version[SERVER_VERSION_LENGTH]; } -ulong server_id = 0; +static char *server_id_str; // needed by net_serv.c ulong bytes_sent = 0L, bytes_received = 0L; @@ -144,6 +145,8 @@ static char *charset= 0; static uint verbose= 0; +static char *ignore_domain_ids_str, *do_domain_ids_str; +static char *ignore_server_ids_str, *do_server_ids_str; static char *start_pos_str, *stop_pos_str; static ulonglong start_position= BIN_LOG_HEADER_SIZE, stop_position= (longlong)(~(my_off_t)0) ; @@ -151,7 +154,10 @@ static ulonglong start_position= BIN_LOG_HEADER_SIZE, #define stop_position_mot ((my_off_t)stop_position) static Binlog_gtid_state_validator *gtid_state_validator= NULL; -static Domain_gtid_event_filter *domain_gtid_filter= NULL; +static Gtid_event_filter *gtid_event_filter= NULL; +static Domain_gtid_event_filter *position_gtid_filter= NULL; +static Domain_gtid_event_filter *domain_id_gtid_filter= NULL; +static Server_gtid_event_filter *server_id_gtid_filter= NULL; static char *start_datetime_str, *stop_datetime_str; static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX; @@ -987,9 +993,16 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, return result; } -static inline my_bool is_gtid_filtering_enabled() +/* + Check if the server id should be excluded from the output. +*/ +static inline my_bool is_server_id_excluded(uint32 server_id) { - return domain_gtid_filter != NULL; + static rpl_gtid server_tester_gtid; + server_tester_gtid.server_id= server_id; + return server_id_gtid_filter == NULL + ? FALSE // No server id filter exists + : server_id_gtid_filter->exclude(&server_tester_gtid); } /** @@ -1061,14 +1074,15 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, glev->count)) goto err; - if (domain_gtid_filter && !domain_gtid_filter->get_num_start_gtids()) + if (position_gtid_filter && + !position_gtid_filter->get_num_start_gtids()) { /* We need to validate the GTID list from --stop-position because we couldn't prove it intrinsically (i.e. using stop > start) */ - rpl_gtid *stop_gtids= domain_gtid_filter->get_stop_gtids(); - size_t n_stop_gtids= domain_gtid_filter->get_num_stop_gtids(); + rpl_gtid *stop_gtids= position_gtid_filter->get_stop_gtids(); + size_t n_stop_gtids= position_gtid_filter->get_num_stop_gtids(); if (gtid_state_validator->verify_stop_state(stderr, stop_gtids, n_stop_gtids)) { @@ -1101,15 +1115,15 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, If the binlog output should be filtered using GTIDs, test the new event group to see if its events should be ignored. */ - if (domain_gtid_filter) + if (gtid_event_filter) { - if (domain_gtid_filter->has_finished()) + if (gtid_event_filter->has_finished()) { retval= OK_STOP; goto end; } - if (!domain_gtid_filter->exclude(&ev_gtid)) + if (!gtid_event_filter->exclude(&ev_gtid)) print_event_info->activate_current_event_group(); else print_event_info->deactivate_current_event_group(); @@ -1124,7 +1138,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, If we don't care about ensuring GTID validity, just delete the auditor object to disable it for future checks. */ - if (gtid_state_validator) + if (gtid_state_validator && print_event_info->is_event_group_active()) { if (!(opt_gtid_strict_mode || verbose >= 3)) { @@ -1180,8 +1194,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, the format_description event so that we can parse subsequent events. */ - if (ev_type != ROTATE_EVENT && - server_id && (server_id != ev->server_id)) + if (ev_type != ROTATE_EVENT && is_server_id_excluded(ev->server_id)) goto end; } if ((ev->when >= stop_datetime) @@ -1769,9 +1782,42 @@ static struct my_option my_options[] = "Print row event positions", &print_row_event_positions, &print_row_event_positions, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, - {"server-id", 0, - "Extract only binlog entries created by the server having the given id.", - &server_id, &server_id, 0, GET_ULONG, + {"ignore-domain-ids", OPT_IGNORE_DOMAIN_IDS, + "A list of positive integers, separated by commas, that form a blacklist " + "of domain ids. Any log event with a GTID that originates from a domain id " + "specified in this list is hidden. Cannot be used with " + "--do-domain-ids. When used with --(ignore|do)-server-ids, the result is the " + "intersection between the two datasets.", + &ignore_domain_ids_str, &ignore_domain_ids_str, 0, GET_STR_ALLOC, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"do-domain-ids", OPT_DO_DOMAIN_IDS, + "A list of positive integers, separated by commas, that form a whitelist " + "of domain ids. Any log event with a GTID that originates from a domain id " + "specified in this list is displayed. Cannot be used with " + "--ignore-domain-ids. When used with --(ignore|do)-server-ids, the result " + "is the intersection between the two datasets.", + &do_domain_ids_str, &do_domain_ids_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, + 0, 0, 0, 0, 0}, + {"ignore-server-ids", OPT_IGNORE_SERVER_IDS, + "A list of positive integers, separated by commas, that form a blacklist " + "of server ids. Any log event originating from a server id " + "specified in this list is hidden. Cannot be used with " + "--do-server-ids. When used with --(ignore|do)-domain-ids, the result is " + "the intersection between the two datasets.", + &ignore_server_ids_str, &ignore_server_ids_str, 0, GET_STR_ALLOC, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"do-server-ids", OPT_DO_SERVER_IDS, + "A list of positive integers, separated by commas, that form a whitelist " + "of server ids. Any log event originating from a server id " + "specified in this list is displayed. Cannot be used with " + "--ignore-server-ids. When used with --(ignore|do)-domain-ids, the result " + "is the intersection between the two datasets. Alias for --server-id.", + &do_server_ids_str, &do_server_ids_str, 0, GET_STR_ALLOC, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"server-id", OPT_SERVER_ID, + "Extract only binlog entries created by the server having the given id. " + "Alias for --do-server-ids.", + &server_id_str, &server_id_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"set-charset", OPT_SET_CHARSET, "Add 'SET NAMES character_set' to the output.", &charset, @@ -1976,9 +2022,31 @@ static void cleanup() my_free(stop_datetime_str); my_free(start_pos_str); my_free(stop_pos_str); + my_free(ignore_domain_ids_str); + my_free(do_domain_ids_str); + my_free(ignore_server_ids_str); + my_free(do_server_ids_str); + my_free(server_id_str); free_root(&glob_root, MYF(0)); - delete domain_gtid_filter; + if (gtid_event_filter) + { + delete gtid_event_filter; + } + else + { + /* + If there was an error during input parsing, gtid_event_filter will not + be set, so we need to ensure the comprising filters are cleaned up + properly. + */ + if (domain_id_gtid_filter) + delete domain_id_gtid_filter; + if (position_gtid_filter) + delete position_gtid_filter; + if (server_id_gtid_filter) + delete server_id_gtid_filter; + } if (gtid_state_validator) delete gtid_state_validator; @@ -1994,6 +2062,89 @@ static void cleanup() DBUG_VOID_RETURN; } +/* + Parse a list of positive numbers separated by commas. + Returns a list of numbers on success, NULL on parsing/resource error +*/ +static uint32 *parse_u32_list(const char *str, size_t str_len, uint32 *n_vals) +{ + const char *str_begin= const_cast(str); + const char *str_end= str_begin + str_len; + const char *p = str_begin; + uint32 len= 0, alloc_len= (uint32) ceil(str_len/2.0); + uint32 *list= NULL; + int err; + + for (;;) + { + uint32 val; + + /* + Set it to the end of the string overall, but when parsing, it will be + moved to the end of the element + */ + char *el_end= (char*) str_begin + str_len; + + if (len >= (((uint32)1 << 28)-1)) + { + my_free(list); + list= NULL; + goto end; + } + + val= (uint32)my_strtoll10(p, &el_end, &err); + if (err) + { + my_free(list); + list= NULL; + goto end; + } + p = el_end; + + if ((!list || len >= alloc_len) && + !(list= + (uint32 *)my_realloc(PSI_INSTRUMENT_ME, list, + (alloc_len= alloc_len*2) * sizeof(uint32), + MYF(MY_FREE_ON_ERROR|MY_ALLOW_ZERO_PTR)))) + return NULL; + list[len++]= val; + + if (el_end == str_end) + break; + if (*p != ',') + { + my_free(list); + return NULL; + } + ++p; + } + *n_vals= len; + +end: + return list; +} + +/* + If multiple different types of Gtid_event_filters are used, the result + should be the intersection between the filter types. +*/ +static void extend_main_gtid_event_filter(Gtid_event_filter *new_filter) +{ + if (gtid_event_filter == NULL) + { + gtid_event_filter= new_filter; + } + else + { + if (gtid_event_filter->get_filter_type() != + Gtid_event_filter::INTERSECTING_GTID_FILTER_TYPE) + gtid_event_filter= + new Intersecting_gtid_event_filter(gtid_event_filter, new_filter); + else + ((Intersecting_gtid_event_filter *) gtid_event_filter) + ->add_filter(new_filter); + } +} static void die() { @@ -2047,6 +2198,110 @@ static my_time_t convert_str_to_timestamp(const char* str) my_system_gmt_sec(&l_time, &dummy_my_timezone, &dummy_in_dst_time_gap); } +/** + Parses a start or stop position argument and populates either + start_position/stop_position (if a log offset) or position_gtid_filter + (if a gtid position) + + @param[in] option_name : Name of the command line option provided (used for + error message) + @param[in] option_val : The user-provided value of the option_name + @param[out] fallback : Pointer to a global variable to set if using log + offsets + @param[in] add_gtid : Function pointer to a class method to add a GTID to a + Gtid_event_filter + @param[in] add_zero_seqno : If using GTID positions, this boolean specifies + if GTIDs with a sequence number of 0 should be added to the filter +*/ +int parse_position_argument( + const char *option_name, char *option_val, ulonglong *fallback, + int (Domain_gtid_event_filter::*add_gtid)(rpl_gtid *), + my_bool add_zero_seqno) +{ + uint32 n_gtids= 0; + rpl_gtid *gtid_list= + gtid_parse_string_to_list(option_val, strlen(option_val), &n_gtids); + + if (gtid_list == NULL) + { + int err= 0; + char *end_ptr= NULL; + /* + No GTIDs specified in position specification. Treat the value + as a singular index. + */ + *fallback= my_strtoll10(option_val, &end_ptr, &err); + + if (err || *end_ptr) + { + // Can't parse the position from the user + sql_print_error("%s argument value is invalid. Should be either a " + "positive integer or GTID.", + option_name); + return 1; + } + } + else if (n_gtids > 0) + { + uint32 gtid_idx; + + if (position_gtid_filter == NULL) + position_gtid_filter= new Domain_gtid_event_filter(); + + for (gtid_idx = 0; gtid_idx < n_gtids; gtid_idx++) + { + rpl_gtid *gtid= >id_list[gtid_idx]; + if ((gtid->seq_no || add_zero_seqno) && + (position_gtid_filter->*add_gtid)(gtid)) + { + my_free(gtid_list); + return 1; + } + } + my_free(gtid_list); + } + else + { + DBUG_ASSERT(0); + } + return 0; +} + +/** + Parses a do/ignore domain/server ids option and populates the corresponding + gtid filter + + @param[in] option_name : Name of the command line option provided (used for + error message) + @param[in] option_value : The user-provided list of domain or server ids + @param[in] filter : The filter to update with the provided domain/server id + @param[in] mode : Specifies whether the list should be a blacklist or + whitelist +*/ +template +int parse_gtid_filter_option( + const char *option_name, char *option_val, T **filter, + Gtid_event_filter::id_restriction_mode mode) +{ + uint32 n_ids= 0; + uint32 *id_list= parse_u32_list(option_val, strlen(option_val), &n_ids); + + if (id_list == NULL) + { + DBUG_ASSERT(n_ids == 0); + sql_print_error( + "Input for %s is invalid. Should be a list of positive integers", + option_name); + return 1; + } + + if (!(*filter)) + (*filter)= new T(); + + int err= (*filter)->set_id_restrictions(id_list, n_ids, mode); + my_free(id_list); + return err; +} extern "C" my_bool get_one_option(const struct my_option *opt, const char *argument, const char *filename) @@ -2234,105 +2489,72 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi case OPT_STOP_POSITION: { /* Stop position was already specified, so reset it and use the new list */ - if (domain_gtid_filter && domain_gtid_filter->get_num_stop_gtids() > 0) - domain_gtid_filter->clear_stop_gtids(); + if (position_gtid_filter && + position_gtid_filter->get_num_stop_gtids() > 0) + position_gtid_filter->clear_stop_gtids(); - uint32 n_stop_gtid_ranges= 0; - rpl_gtid *stop_gtids= gtid_parse_string_to_list( - stop_pos_str, strlen(stop_pos_str), &n_stop_gtid_ranges); - if (stop_gtids == NULL) - { - int err= 0; - char *end_ptr= NULL; - /* - No GTIDs specified in OPT_STOP_POSITION specification. Treat the value - as a singular index. - */ - stop_position= my_strtoll10(stop_pos_str, &end_ptr, &err); - - if (err || *end_ptr) - { - // Can't parse the position from the user - sql_print_error("Stop position argument value is invalid. Should be " - "either a positive integer or GTID."); - return 1; - } - } - else if (n_stop_gtid_ranges > 0) - { - uint32 gtid_idx; - - if (domain_gtid_filter == NULL) - domain_gtid_filter= new Domain_gtid_event_filter(); - - for (gtid_idx = 0; gtid_idx < n_stop_gtid_ranges; gtid_idx++) - { - rpl_gtid *stop_gtid= &stop_gtids[gtid_idx]; - if (domain_gtid_filter->add_stop_gtid(stop_gtid)) - { - my_free(stop_gtids); - return 1; - } - } - my_free(stop_gtids); - } - else - { - DBUG_ASSERT(0); - } + if (parse_position_argument( + "--stop-position", stop_pos_str, &stop_position, + &Domain_gtid_event_filter::add_stop_gtid, TRUE)) + return 1; break; } case 'j': { /* Start position was already specified, so reset it and use the new list */ - if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) - domain_gtid_filter->clear_start_gtids(); + if (position_gtid_filter && + position_gtid_filter->get_num_start_gtids() > 0) + position_gtid_filter->clear_start_gtids(); - uint32 n_start_gtid_ranges= 0; - rpl_gtid *start_gtids= gtid_parse_string_to_list( - start_pos_str, strlen(start_pos_str), &n_start_gtid_ranges); - - if (start_gtids == NULL) - { - int err= 0; - char *end_ptr= NULL; - /* - No GTIDs specified in OPT_START_POSITION specification. Treat the value - as a singular index. - */ - start_position= my_strtoll10(start_pos_str, &end_ptr, &err); - - if (err || *end_ptr) - { - // Can't parse the position from the user - sql_print_error("Start position argument value is invalid. Should be " - "either a positive integer or GTID."); - return 1; - } - } - else if (n_start_gtid_ranges > 0) - { - uint32 gtid_idx; - - if (domain_gtid_filter == NULL) - domain_gtid_filter= new Domain_gtid_event_filter(); - - for (gtid_idx = 0; gtid_idx < n_start_gtid_ranges; gtid_idx++) - { - rpl_gtid *start_gtid= &start_gtids[gtid_idx]; - if (start_gtid->seq_no && - domain_gtid_filter->add_start_gtid(start_gtid)) - { - my_free(start_gtids); - return 1; - } - } - my_free(start_gtids); - } - else - { - DBUG_ASSERT(0); - } + if (parse_position_argument( + "--start-position", start_pos_str, &start_position, + &Domain_gtid_event_filter::add_start_gtid, FALSE)) + return 1; + break; + } + case OPT_IGNORE_DOMAIN_IDS: + { + if (parse_gtid_filter_option( + "--ignore-domain-ids", ignore_domain_ids_str, + &domain_id_gtid_filter, + Gtid_event_filter::id_restriction_mode::BLACKLIST_MODE)) + return 1; + break; + } + case OPT_DO_DOMAIN_IDS: + { + if (parse_gtid_filter_option( + "--do-domain-ids", do_domain_ids_str, + &domain_id_gtid_filter, + Gtid_event_filter::id_restriction_mode::WHITELIST_MODE)) + return 1; + break; + } + case OPT_IGNORE_SERVER_IDS: + { + if (parse_gtid_filter_option( + "--ignore-server-ids", ignore_server_ids_str, + &server_id_gtid_filter, + Gtid_event_filter::id_restriction_mode::BLACKLIST_MODE)) + return 1; + break; + } + case OPT_DO_SERVER_IDS: + { + if (parse_gtid_filter_option( + "--do-server-ids", do_server_ids_str, + &server_id_gtid_filter, + Gtid_event_filter::id_restriction_mode::WHITELIST_MODE)) + return 1; + break; + } + case OPT_SERVER_ID: + { + if (parse_gtid_filter_option( + "--server-id", server_id_str, + &server_id_gtid_filter, + Gtid_event_filter::id_restriction_mode::WHITELIST_MODE)) + return 1; break; } case '?': @@ -2346,7 +2568,6 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi return 0; } - static int parse_args(int *argc, char*** argv) { int ho_error; @@ -2376,9 +2597,10 @@ static int parse_args(int *argc, char*** argv) */ gtid_state_validator= new Binlog_gtid_state_validator(); - if (domain_gtid_filter) + if (position_gtid_filter) { - if (opt_gtid_strict_mode && domain_gtid_filter->validate_window_filters()) + if (opt_gtid_strict_mode && + position_gtid_filter->validate_window_filters()) { /* In strict mode, if any --start/stop-position GTID ranges are invalid, @@ -2387,17 +2609,24 @@ static int parse_args(int *argc, char*** argv) */ die(); } + extend_main_gtid_event_filter(position_gtid_filter); /* GTIDs before a start position shouldn't be validated, so we initialize the stream auditor to only monitor GTIDs after these positions. */ - size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); - rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + size_t n_start_gtids= position_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= position_gtid_filter->get_start_gtids(); gtid_state_validator->initialize_start_gtids(start_gtids, n_start_gtids); my_free(start_gtids); } + if(domain_id_gtid_filter) + extend_main_gtid_event_filter(domain_id_gtid_filter); + + if(server_id_gtid_filter) + extend_main_gtid_event_filter(server_id_gtid_filter); + return 0; } @@ -2479,8 +2708,9 @@ static Exit_status dump_log_entries(const char* logname) if (!print_event_info.init_ok()) return ERROR_STOP; - if (domain_gtid_filter) + if (position_gtid_filter || domain_id_gtid_filter) print_event_info.enable_event_group_filtering(); + /* Set safe delimiter, to dump things like CREATE PROCEDURE safely @@ -2582,7 +2812,8 @@ static Exit_status check_master_version() goto err; } - if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) + if (position_gtid_filter && + position_gtid_filter->get_num_start_gtids() > 0) { char str_buf[256]; String query_str(str_buf, sizeof(str_buf), system_charset_info); @@ -2590,8 +2821,8 @@ static Exit_status check_master_version() query_str.append(STRING_WITH_LEN("SET @slave_connect_state='"), system_charset_info); - size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); - rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + size_t n_start_gtids= position_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= position_gtid_filter->get_start_gtids(); for (size_t gtid_idx = 0; gtid_idx < n_start_gtids; gtid_idx++) { @@ -3539,7 +3770,7 @@ int main(int argc, char** argv) "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n"); fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;\n"); - if (is_gtid_filtering_enabled()) + if (gtid_event_filter) { fprintf(result_file, "/*!100001 SET @@SESSION.SERVER_ID=@@GLOBAL.SERVER_ID */;\n" diff --git a/man/mysqlbinlog.1 b/man/mysqlbinlog.1 index 7ad8c714490..5430c80ddc6 100644 --- a/man/mysqlbinlog.1 +++ b/man/mysqlbinlog.1 @@ -882,11 +882,78 @@ quote the value (e.g. \fB--rewrite-db="oldname->newname"\fR\&. .sp -1 .IP \(bu 2.3 .\} +.\" mysqlbinlog: ignore-domain-ids option +.\" ignore-domain-ids option: mysqlbinlog +\fB\-\-ignore\-domain\-ids=\fR\fB\fIIDs\fR\fR +.sp +Hide events which are a part of any stream identified by the domain ID +list \fIIDs\fR\&. Cannot be used with \-\-do\-domain\-ids\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysqlbinlog: do-domain-ids option +.\" do-domain-ids option: mysqlbinlog +\fB\-\-do\-domain\-ids=\fR\fB\fIIDs\fR\fR +.sp +Display only the events which exist in a stream identified by \fIIDs\fR\&. +Cannot be used with \-\-ignore\-domain\-ids\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysqlbinlog: ignore-server-ids option +.\" ignore-server-ids option: mysqlbinlog +\fB\-\-ignore\-server\-ids=\fR\fB\fIIDs\fR\fR +.sp +Hide events which originated from any server whose ID is specified in +\fIIDs\fR\&. Cannot be used with \-\-do\-server\-ids\ or \-\-server\-id\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysqlbinlog: do-server-ids option +.\" do-server-ids option: mysqlbinlog +\fB\-\-do\-server\-ids=\fR\fB\fIIDs\fR\fR +.sp +Display only the events that originated from a server identified in +\fIIDs\fR\&. Cannot be used with \-\-ignore\-server\-ids, and +is an alias for \-\-server\-id\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} .\" mysqlbinlog: server-id option .\" server-id option: mysqlbinlog \fB\-\-server\-id=\fR\fB\fIid\fR\fR .sp Display only those events created by the server having the given server ID\&. +Cannot be used with \-\-ignore\-server\-ids, and is an alias for +\-\-do\-server\-ids\&. .RE .sp .RS 4 diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_do_server_ids.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_do_server_ids.result new file mode 100644 index 00000000000..0f5d1694522 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_do_server_ids.result @@ -0,0 +1,161 @@ +############################### +# Test Setup +############################### +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (3); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 values (4); +SET @@session.server_id= 3; +SET timestamp=@a+1; +CREATE TABLE t4 (a int); +SET timestamp=@a+2; +INSERT INTO t4 values (5); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 1; +CREATE TABLE t5 (a int); +INSERT INTO t5 values (6); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (2); +FLUSH LOGS; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 2; +CREATE TABLE t6 (a int); +INSERT INTO t6 values (1); +FLUSH LOGS; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; +DROP TABLE t6; +RESET MASTER; +############################### +# Test Cases +############################### +# +# Test Case 1) --do-server-ids with a single server id limits output +# to that single server +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2 | MYSQL +DROP TABLE t2; +# +# Test Case 2) --do-server-ids with multiple server ids limits output +# to the provided servers +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2,3 | MYSQL +DROP TABLE t2; +DROP TABLE t4; +# +# Test Case 3) --do-server-ids when combined with --do-domain-ids should +# intersect the results +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --do-domain-ids=0 | MYSQL +DROP TABLE t1; +# +# Test Case 4) --do-server-ids when combined with --ignore-domain-ids should +# intersect the results +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 | MYSQL +DROP TABLE t3; +DROP TABLE t5; +# +# Test Case 5) --do-server-ids when combined with a GTID range should +# intersect the results +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --stop-position=0-1-4 | MYSQL +DROP TABLE t1; +# +# Test Case 6) --do-server-ids when combined with both --ignore-domain-ids +# and a GTID range should intersect all results +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 --start-position=1-1-0 | MYSQL +DROP TABLE t3; +DROP TABLE t5; +# +# Test Case 7) --do-server-ids when combined with both --do-domain-ids and +# a GTID range should intersect all results +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2 --do-domain-ids=0 --start-position=0-1-0 | MYSQL +DROP TABLE t2; +# +# Test Case 8) --do-server-ids and --offset= skips n events after the +# first GTID is found +CREATE TABLE t4 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --offset=5 --do-server-ids=3 --do-domain-ids=1 | MYSQL +DROP TABLE t4; +# +# Test Case 9) --do-server-ids with --start-datetime= where T occurs +# after the first GTID is found results in no events before T +CREATE TABLE t4 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-datetime="1970-01-21 15:32:24" --do-server-ids=3 --do-domain-ids=1 | MYSQL +DROP TABLE t4; +# +# Test Case 10) --do-server-ids works with --read-from-remote-server +# Setup test specific data +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 VALUES (1); +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 VALUES (1); +SET @@session.server_id= 1; +DROP TABLE t1; +DROP TABLE t2; +# MYSQL_BINLOG BINLOG_FILENAME --read-from-remote-server --do-server-ids=2 | MYSQL +DROP TABLE t2; +# +# Test Case 11) --do-server-ids works over multiple binary log input +# files +# MYSQL_BINLOG BINLOG_FILE_PARAM BINLOG_FILE_PARAM2 --do-server-ids=2 | MYSQL +DROP TABLE t2; +DROP TABLE t6; +# +# Test Case 12) --do-server-ids re-specifications should override +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --do-server-ids=2 | MYSQL +DROP TABLE t2; +# +# Test Case 13) --do-server-ids and --server-id should be aliases and +# a re-specification of one should override the former +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --server-id=2 | MYSQL +DROP TABLE t2; +# +# Test Case 14) --ignore-server-ids re-specifications should override +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=2 --ignore-server-ids=1,3 | MYSQL +DROP TABLE t2; +# +# Test Case 15) --do-domain-ids re-specifications should override +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=1 --do-domain-ids=0 | MYSQL +DROP TABLE t1,t2; +# +# Test Case 16) --ignore-domain-ids re-specifications should override +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=0 --ignore-domain-ids=1,2 | MYSQL +DROP TABLE t1,t2; +############################## +# Error Cases +############################## +# +# Error Case 1: +# --ignore-server-ids and --do-server-ids both specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-server-ids=2 +# +# Error Case 2: +# Invalid server ID number provided +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=-1 +############################## +# Cleanup +############################## +SET @@global.gtid_domain_id= 0; +SET @@global.server_id= 1; +# End of tests diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_do_server_ids.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_do_server_ids.test new file mode 100644 index 00000000000..9f32e2db071 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_do_server_ids.test @@ -0,0 +1,477 @@ +# +# Purpose: +# +# This test validates that the option --do-server-ids for the mariadb-binlog +# command line tool correctly filters events by server id. +# +# +# Methodology: +# +# This test invokes mariadb-binlog using combinations of --do-server-ids with +# various combinations of other parameters to ensure it correctly filters by +# server id. Specifically, the following test cases are validated: +# Test Case 1) --do-server-ids with a single server id limits output to that +# single server +# Test Case 2) --do-server-ids with multiple server ids limits output to the +# provided servers +# Test Case 3) --do-server-ids when combined with --do-domain-ids should +# intersect the results +# Test Case 4) --do-server-ids when combined with --ignore-domain-ids should +# intersect the results +# Test Case 5) --do-server-ids when combined with a GTID range should +# intersect the results +# Test Case 6) --do-server-ids when combined with both --ignore-domain-ids +# and a GTID range should intersect all results +# Test Case 7) --do-server-ids when combined with both --do-domain-ids and +# a GTID range should intersect all results +# Test Case 8) --do-server-ids and --offset= skips n events after the +# first GTID is found +# Test Case 9) --do-server-ids with --start-datetime= where T occurs +# after the first GTID is found results in no events before T +# Test Case 10) --do-server-ids works with --read-from-remote-server +# Test Case 11) --do-server-ids works over multiple binary log input files +# Test Case 12) --do-server-ids re-specifications should override previous +# ones +# Test Case 13) --do-server-ids and --server-id should be aliases and a +# re-specification of one should override the former +# Test Case 14) --ignore-server-ids re-specifications should override +# previous ones +# Test Case 15) --do-domain-ids re-specifications should override previous +# ones +# Test Case 16) --ignore-domain-ids re-specifications should override +# previous ones +# +# Additionally, this test validates the following error scenarios: +# Error Case 1) --do-server-ids and --ignore-server-ids cannot be specified +# together +# Error Case 2) Invalid server ID number provided +# +# +# References: +# +# MDEV-20119: Implement the --do-domain-ids, --ignore-domain-ids, and +# --ignore-server-ids options for mysqlbinlog +# + +--source include/have_log_bin.inc + +--echo ############################### +--echo # Test Setup +--echo ############################### + + +# Save old state +let $ORIG_GTID_DOMAIN_ID = `select @@session.gtid_domain_id`; +let $ORIG_SERVER_ID = `select @@session.server_id`; + +# Configure test variables +--let $MYSQLD_DATADIR=`select @@datadir` + +--let table_inconsistent_err= "table data is inconsistent after replaying binlog events"; +--let table_exists_error= "table exists but binlog playback should have excluded its creation"; + +# Initialize test data +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; +RESET MASTER; + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); + +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (3); +--let t2_checksum= `CHECKSUM TABLE t2` + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 values (4); +--let t3_checksum= `CHECKSUM TABLE t3` + +SET @@session.server_id= 3; +SET timestamp=@a+1; +CREATE TABLE t4 (a int); +SET timestamp=@a+2; +INSERT INTO t4 values (5); +--let t4_checksum= `CHECKSUM TABLE t4` + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (1); +--let t1_partial_checksum= `CHECKSUM TABLE t1` + +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 1; +CREATE TABLE t5 (a int); +INSERT INTO t5 values (6); +--let t5_checksum= `CHECKSUM TABLE t5` + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (2); +--let t1_checksum= `CHECKSUM TABLE t1` + +FLUSH LOGS; + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 2; +CREATE TABLE t6 (a int); +INSERT INTO t6 values (1); +--let t6_checksum= `CHECKSUM TABLE t6` +FLUSH LOGS; + +--let BINLOG_FILENAME= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--let BINLOG_FILENAME2= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--let BINLOG_FILE_PARAM= $MYSQLD_DATADIR/$BINLOG_FILENAME.orig +--let BINLOG_FILE_PARAM2= $MYSQLD_DATADIR/$BINLOG_FILENAME2.orig +--copy_file $MYSQLD_DATADIR/$BINLOG_FILENAME $BINLOG_FILE_PARAM +--copy_file $MYSQLD_DATADIR/$BINLOG_FILENAME2 $BINLOG_FILE_PARAM2 + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; +DROP TABLE t6; +RESET MASTER; + +--echo ############################### +--echo # Test Cases +--echo ############################### + +--echo # +--echo # Test Case 1) --do-server-ids with a single server id limits output +--echo # to that single server +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=2 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + + +--echo # +--echo # Test Case 2) --do-server-ids with multiple server ids limits output +--echo # to the provided servers +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2,3 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=2,3 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if ($t4_checksum != `CHECKSUM TABLE t4`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; +DROP TABLE t4; + + +--echo # +--echo # Test Case 3) --do-server-ids when combined with --do-domain-ids should +--echo # intersect the results +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --do-domain-ids=0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --do-domain-ids=0 | $MYSQL +if ($t1_checksum != `CHECKSUM TABLE t1`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t2','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t1; + + +--echo # +--echo # Test Case 4) --do-server-ids when combined with --ignore-domain-ids should +--echo # intersect the results +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 | $MYSQL +if ($t3_checksum != `CHECKSUM TABLE t3`) +{ + die $table_inconsistent_err; +} +if ($t5_checksum != `CHECKSUM TABLE t5`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t2','t4','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t3; +DROP TABLE t5; + + +--echo # +--echo # Test Case 5) --do-server-ids when combined with a GTID range should +--echo # intersect the results +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --stop-position=0-1-4 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --stop-position=0-1-4 | $MYSQL +if ($t1_partial_checksum != `CHECKSUM TABLE t1`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t2','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t1; + + +--echo # +--echo # Test Case 6) --do-server-ids when combined with both --ignore-domain-ids +--echo # and a GTID range should intersect all results +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 --start-position=1-1-0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --ignore-domain-ids=0 --start-position=1-1-0 | $MYSQL +if ($t3_checksum != `CHECKSUM TABLE t3`) +{ + die $table_inconsistent_err; +} +if ($t5_checksum != `CHECKSUM TABLE t5`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t2','t4','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t3; +DROP TABLE t5; + + +--echo # +--echo # Test Case 7) --do-server-ids when combined with both --do-domain-ids and +--echo # a GTID range should intersect all results +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=2 --do-domain-ids=0 --start-position=0-1-0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=2 --do-domain-ids=0 --start-position=0-1-0 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + + + +--echo # +--echo # Test Case 8) --do-server-ids and --offset= skips n events after the +--echo # first GTID is found + +# t4 needs to be specified because its creation should be skipped from +# --offset specification +CREATE TABLE t4 (a int); + +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --offset=5 --do-server-ids=3 --do-domain-ids=1 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --offset=5 --do-server-ids=3 --do-domain-ids=1 | $MYSQL +if ($t4_checksum != `CHECKSUM TABLE t4`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t2','t3','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t4; + + +--echo # +--echo # Test Case 9) --do-server-ids with --start-datetime= where T occurs +--echo # after the first GTID is found results in no events before T + +# t4 needs to be specified because its creation should be skipped from +# --start-datetime specification +CREATE TABLE t4 (a int); + +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-datetime="1970-01-21 15:32:24" --do-server-ids=3 --do-domain-ids=1 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-datetime="1970-01-21 15:32:24" --do-server-ids=3 --do-domain-ids=1 | $MYSQL +if ($t4_checksum != `CHECKSUM TABLE t4`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t2','t3','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t4; + + +--echo # +--echo # Test Case 10) --do-server-ids works with --read-from-remote-server + +--echo # Setup test specific data +RESET MASTER; + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 VALUES (1); + +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 VALUES (1); +--let t11_t2_checksum= `CHECKSUM TABLE t2` + +SET @@session.server_id= 1; +DROP TABLE t1; +DROP TABLE t2; + +--echo # MYSQL_BINLOG BINLOG_FILENAME --read-from-remote-server --do-server-ids=2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILENAME --read-from-remote-server --do-server-ids=2 | $MYSQL +if ($t11_t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + + +--echo # +--echo # Test Case 11) --do-server-ids works over multiple binary log input +--echo # files +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM BINLOG_FILE_PARAM2 --do-server-ids=2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM $BINLOG_FILE_PARAM2 --do-server-ids=2 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if ($t6_checksum != `CHECKSUM TABLE t6`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5')`) +{ + die $table_exists_error; +} +DROP TABLE t2; +DROP TABLE t6; + +--echo # +--echo # Test Case 12) --do-server-ids re-specifications should override +--echo # previous ones +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --do-server-ids=2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --do-server-ids=2 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 13) --do-server-ids and --server-id should be aliases and +--echo # a re-specification of one should override the former +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --server-id=2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --server-id=2 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 14) --ignore-server-ids re-specifications should override +--echo # previous ones +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=2 --ignore-server-ids=1,3 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --ignore-server-ids=2 --ignore-server-ids=1,3 | $MYSQL +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t1','t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 15) --do-domain-ids re-specifications should override +--echo # previous ones +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=1 --do-domain-ids=0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-domain-ids=1 --do-domain-ids=0 | $MYSQL +if ($t1_checksum != `CHECKSUM TABLE t1`) +{ + die $table_inconsistent_err; +} +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t1,t2; + +--echo # +--echo # Test Case 16) --ignore-domain-ids re-specifications should override +--echo # previous ones +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=0 --ignore-domain-ids=1,2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --ignore-domain-ids=0 --ignore-domain-ids=1,2 | $MYSQL +if ($t1_checksum != `CHECKSUM TABLE t1`) +{ + die $table_inconsistent_err; +} +if ($t2_checksum != `CHECKSUM TABLE t2`) +{ + die $table_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name IN ('t3','t4','t5','t6')`) +{ + die $table_exists_error; +} +DROP TABLE t1,t2; + +--echo ############################## +--echo # Error Cases +--echo ############################## + +--echo # +--echo # Error Case 1: +--echo # --ignore-server-ids and --do-server-ids both specified +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=1 --ignore-server-ids=2 +--error 1 +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=1 --ignore-server-ids=2 + +--echo # +--echo # Error Case 2: +--echo # Invalid server ID number provided +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --do-server-ids=-1 +--error 1 +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --do-server-ids=-1 + +--echo ############################## +--echo # Cleanup +--echo ############################## +--eval SET @@global.gtid_domain_id= $ORIG_GTID_DOMAIN_ID +--eval SET @@global.server_id= $ORIG_SERVER_ID + +--remove_file $BINLOG_FILE_PARAM +--remove_file $BINLOG_FILE_PARAM2 + +--echo # End of tests diff --git a/mysql-test/suite/rpl/include/mysqlbinlog_slave_consistency.inc b/mysql-test/suite/rpl/include/mysqlbinlog_slave_consistency.inc new file mode 100644 index 00000000000..d8476df9f53 --- /dev/null +++ b/mysql-test/suite/rpl/include/mysqlbinlog_slave_consistency.inc @@ -0,0 +1,201 @@ +# This file provides the structure to run a single test that ensures the +# mariadb-binlog command line tool is consistent with replicas for event +# filtering. The test is configured using the following input parameters, where +# each is nullable (i.e. it will not be used to configure mariadb-binlog or +# the replica). +# +# param $do_domain_ids : A list of domain ids to include in replication +# param $ignore_domain_ids : A list of domain ids to exclude from replication +# param $ignore_server_ids : A list of server ids to exclude from replication +# param $start_position : The GTID positions to begin replication from in +# the specified domains +# param $stop_position : The GTID positions that mark the end of an event +# stream in a particular domain +# +# param $con1 : The connection name of the primary server +# param $con2 : The connection name of the replica server +# param $strict_mode : Uses input and checks for out of order GTIDs +# param $strict_mode_err : A boolean that provides expectations for strict +# mode to error +# param $slave_sql_errno : Expected error number of the slave SQL thread + + +--let $include_filename= mysqlbinlog_slave_consistency.inc +--source include/begin_include_file.inc + +--enable_query_log + +if (!$con1) +{ + --let $con1=master +} +if (!$con2) +{ + --let $con2=slave +} + +if (!$strict_mode) +{ + --connection $con2 + set @@global.gtid_strict_mode=0; + --let $sql_input_file=include/sql_multisource.inc +} + +if ($strict_mode) +{ + --connection $con2 + set @@global.gtid_strict_mode=1; + --let $sql_input_file=include/sql_out_of_order_gtid.inc +} + +--connection $con2 +--source include/stop_slave.inc + +--connection $con1 +--echo # Populating $con1 data +--source $sql_input_file + +--let $MYSQLD_DATADIR=`select @@datadir` +--let $MYSQLBINLOG_STDERR=$MYSQLD_DATADIR/mysqlbinlog_stderr.out +--let BINLOG_FILENAME= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--let BINLOG_FILE_PARAM= $MYSQLD_DATADIR/$BINLOG_FILENAME.orig +--copy_file $MYSQLD_DATADIR/$BINLOG_FILENAME $BINLOG_FILE_PARAM + +--connection $con2 +--let $msbl_args= +if (`SELECT strcmp("$start_position","") != 0`) +{ + eval set global gtid_slave_pos="$start_position"; + --let $msbl_args= $msbl_args --start-position=$start_position +} + +--let $cm_args= MASTER_USE_GTID=slave_pos +if (`SELECT strcmp("$do_domain_ids","") != 0`) +{ + --let $cm_args= $cm_args, DO_DOMAIN_IDS=($do_domain_ids) + --let $msbl_args= $msbl_args --do-domain-ids=$do_domain_ids +} +if (`SELECT strcmp("$ignore_domain_ids","") != 0`) +{ + --let $cm_args= $cm_args, IGNORE_DOMAIN_IDS=($ignore_domain_ids) + --let $msbl_args= $msbl_args --ignore-domain-ids=$ignore_domain_ids +} +if (`SELECT strcmp("$ignore_server_ids","") != 0`) +{ + --let $cm_args= $cm_args, IGNORE_SERVER_IDS=($ignore_server_ids) + --let $msbl_args= $msbl_args --ignore-server-ids=$ignore_server_ids +} +if ($strict_mode) +{ + --let $msbl_args= $msbl_args --gtid-strict-mode +} +eval CHANGE MASTER TO $cm_args; + +--let $start_slave_args= +if (`SELECT strcmp("$stop_position","") != 0`) +{ + --let $start_slave_args= UNTIL master_gtid_pos="$stop_position" + --let $msbl_args= $msbl_args --stop-position=$stop_position +} + +eval START SLAVE $start_slave_args; + +if ($slave_sql_errno) +{ + --echo # $con2 SQL Thread error expected - waiting for errno $slave_sql_errno + --source include/wait_for_slave_sql_error.inc +} + +# If we are not expecting an error, wait for con2 to catch up +if (!$slave_sql_errno) +{ + --echo # No $con2 error expecting - waiting for $con2 to catch up to $con1 + + # Stop position was not specified + if (`SELECT strcmp("$stop_position","") = 0`) + { + --source include/wait_for_slave_to_start.inc + --echo # Wait for $con2 IO thread to catch up + --let $wait_condition= SELECT STATE="Waiting for master to send event" from information_schema.PROCESSLIST where COMMAND="Slave_IO" + --source include/wait_condition.inc + + --echo # Wait for $con2 SQL thread to catch up + --let $wait_condition= SELECT STATE="Slave has read all relay log; waiting for more updates" from information_schema.PROCESSLIST where COMMAND="Slave_SQL" + --source include/wait_condition.inc + + } + + # Stop position was specified + if (`SELECT strcmp("$stop_position","") != 0`) + { + --echo # Because there is a stop position we wait for all events to process + --echo # and $con2 to automatically stop + --source include/wait_for_slave_to_stop.inc + } +} + +--echo # Stop $con2 so it stops receiving $con1 events. +--source include/stop_slave.inc + +--connection $con1 +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM $msbl_args 2> MYSQLBINLOG_STDERR | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM $msbl_args 2> $MYSQLBINLOG_STDERR | $MYSQL + +--source include/rpl_check_table_consistency.inc + +if ($strict_mode) +{ + --echo # Strict mode enabled - checking mysqlbinlog error output for out + --echo # of order GTIDs + --let SEARCH_FILE=$MYSQLBINLOG_STDERR + --let SEARCH_PATTERN=Found out of order GTID + if ($strict_mode_err) + { + --echo # Expecting to find out of order GTID error.. + } + if (!$strict_mode_err) + { + --echo # Not expecting to find out of order GTID error.. + } + --source include/search_pattern_in_file.inc +} + +--echo # Test finished - resetting $con1 and $con2.. +--connection $con2 +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); + +--connection $con1 +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +--source include/save_master_gtid.inc + +--connection $con2 +--source include/start_slave.inc +--source include/wait_for_slave_to_start.inc +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; + +--connection $con1 +RESET MASTER; + +--connection $con2 +if ($strict_mode) +{ + set @@global.gtid_strict_mode=0; +} +--source include/start_slave.inc + +--connection $con1 +--remove_file $BINLOG_FILE_PARAM +--remove_file $MYSQLBINLOG_STDERR + +--let $include_filename= mysqlbinlog_slave_consistency.inc +--source include/end_include_file.inc diff --git a/mysql-test/suite/rpl/include/rpl_check_table_consistency.inc b/mysql-test/suite/rpl/include/rpl_check_table_consistency.inc new file mode 100644 index 00000000000..957a5d525a7 --- /dev/null +++ b/mysql-test/suite/rpl/include/rpl_check_table_consistency.inc @@ -0,0 +1,63 @@ +# This file provides logic to ensure that all tables in a database are the +# same between two connections. +# +# param $check_db : The name of the database to validate all tables are the +# same within (test by default) +# param $con1 : The connection name of the primary server, defaults to +# master +# param $con2 : The connection name of the replica server, defaults to +# slave + +--let $include_filename= rpl_check_table_consistency.inc +--source include/begin_include_file.inc + +if (!$con1) +{ + --let $con1= master +} +if (!$con2) +{ + --let $con2= slave +} +if (!$check_db) +{ + --let $check_db= test +} + +--connection $con2 +--let $n_tables= `select count(*) from information_schema.tables WHERE table_schema = '$check_db'` + +--echo # Checking consistency of '$check_db' database tables between $con1 and $con2 + +--connection $con1 +--let $c1_n_tables= `select count(*) from information_schema.tables WHERE table_schema = '$check_db'` +if (`SELECT $c1_n_tables != $n_tables`) +{ + die "$con1 had $c1_n_tables tables but $con2 had $n_tables after binlog replay"; +} +--echo # Both servers have $n_tables tables + +--let $ctr= 1 +--echo # Verifying integrity of tables.. +while($ctr <= $n_tables) +{ + --let $cksum_tbl= query_get_value(SELECT table_name FROM information_schema.tables WHERE table_schema = 'test' ORDER BY table_name ASC, table_name, $ctr) + --connection $con1 + --let $c1_cksum= `CHECKSUM TABLE $cksum_tbl` + --connection $con2 + --let $c2_cksum= `CHECKSUM TABLE $cksum_tbl` + + if ($c1_cksum != $c2_cksum) + { + die "Table $cksum_tbl differs between connections $con1 and $con2"; + } + if ($c1_cksum == $c2_cksum) + { + --echo # $cksum_tbl is equivalent on $con1 and $con2 + } + --let $ctr= `SELECT $ctr+1` +} +--echo # All tables are consistent + +--let $include_filename= rpl_check_table_consistency.inc +--source include/end_include_file.inc diff --git a/mysql-test/suite/rpl/include/sql_multisource.inc b/mysql-test/suite/rpl/include/sql_multisource.inc new file mode 100644 index 00000000000..761b68421d9 --- /dev/null +++ b/mysql-test/suite/rpl/include/sql_multisource.inc @@ -0,0 +1,45 @@ +# Populate the active connection server with events that come from varying +# domain and server ids + +--disable_query_log + +# Save old state +let $ORIG_GTID_DOMAIN_ID = `select @@session.gtid_domain_id`; +let $ORIG_SERVER_ID = `select @@session.server_id`; + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); + +SET @@session.server_id= 3; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (3); + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 values (4); + +SET @@session.server_id= 4; +CREATE TABLE t4 (a int); +INSERT INTO t4 values (5); + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (1); + +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 1; +CREATE TABLE t5 (a int); +INSERT INTO t5 values (6); + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (2); + +FLUSH LOGS; + +--eval SET @@session.gtid_domain_id= $ORIG_GTID_DOMAIN_ID +--eval SET @@session.server_id= $ORIG_SERVER_ID + +--enable_query_log diff --git a/mysql-test/suite/rpl/include/sql_out_of_order_gtid.inc b/mysql-test/suite/rpl/include/sql_out_of_order_gtid.inc new file mode 100644 index 00000000000..82291a887e9 --- /dev/null +++ b/mysql-test/suite/rpl/include/sql_out_of_order_gtid.inc @@ -0,0 +1,43 @@ +# SQL file with out of order GTIDs coming from various domains and servers + +--disable_query_log + +# Save old state +let $ORIG_GTID_DOMAIN_ID = `select @@session.gtid_domain_id`; +let $ORIG_SERVER_ID = `select @@session.server_id`; + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); + +SET @@session.server_id= 3; +CREATE TABLE t2 (a int); +SET @@session.gtid_seq_no= 6; +INSERT INTO t2 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t2 values (1); +SET @@session.gtid_seq_no= 7; +INSERT INTO t2 values (3); + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t3 values (3); +SET @@session.gtid_seq_no= 3; +INSERT INTO t3 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t3 values (4); + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 values (2); + +FLUSH LOGS; + +--eval SET @@session.gtid_domain_id= $ORIG_GTID_DOMAIN_ID +--eval SET @@session.server_id= $ORIG_SERVER_ID + +--enable_query_log diff --git a/mysql-test/suite/rpl/r/rpl_mysqlbinlog_slave_consistency.result b/mysql-test/suite/rpl/r/rpl_mysqlbinlog_slave_consistency.result new file mode 100644 index 00000000000..858ee7648a8 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_mysqlbinlog_slave_consistency.result @@ -0,0 +1,1371 @@ +include/master-slave.inc +[connection master] +connection slave; +SET sql_log_bin=0; +call mtr.add_suppression("Slave: An attempt was made.*"); +call mtr.add_suppression("Both DO_DOMAIN_IDS & IGNORE_DOMAIN_IDS lists can't be non-empty at the same time"); +SET sql_log_bin=1; +include/stop_slave.inc +connection master; +RESET MASTER; +connection slave; +include/start_slave.inc +# +# +# Test Case 1: Base case to ensure that mariadb-binlog and replica +# are consistent without any filtering +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 5 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t3 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 2: Ensure filtering by a single id in --do-domain-ids is +# consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=0 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 2 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 3: Ensure filtering by multiple ids in --do-domain-ids is +# consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0,1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=0,1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 4 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t3 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# Test Case 4: Ensure filtering by a single id in --ignore-domain-ids +# is consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=0 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 3 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t3 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 5: Ensure filtering by multiple ids in --ignore-domain-ids +# is consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0,1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=0,1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t3,test.t4' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# Test Case 6: Ensure filtering by a single id in --ignore-server-ids +# is consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 2 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 7: Ensure filtering by multiple ids in --ignore-server-ids +# is consistent between mariadb-binlog and replica +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(0,1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=0,1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 2 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 8: Ensure stop position consistency +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +START SLAVE UNTIL master_gtid_pos="1-1-2"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-1-2 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t3 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 9: Ensure start position consistency +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="1-4-2"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-4-2 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 4 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t3' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 10: Ensure consistency when filtering by both +# --do-domain-ids and --ignore-server-ids +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_SERVER_IDS=(1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=0 --ignore-server-ids=1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 11: Ensure consistency when filtering by both +# --ignore-domain-ids and --ignore-server-ids +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(1,2), IGNORE_SERVER_IDS=(1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=1,2 --ignore-server-ids=1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 12: Ensure consistency when filtering by +# --do-domain-ids with a stop position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0); +START SLAVE UNTIL master_gtid_pos="0-1-4"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=0 --stop-position=0-1-4 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 2 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 13: Ensure consistency when filtering by +# --ignore-domain-ids with a stop position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0); +START SLAVE UNTIL master_gtid_pos="0-1-4"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-domain-ids=0 --stop-position=0-1-4 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 0 tables +# Verifying integrity of tables.. +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 14: Ensure consistency when filtering by +# --ignore-server-ids with a stop position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(3); +START SLAVE UNTIL master_gtid_pos="0-1-4"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=3 --stop-position=0-1-4 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 15: Ensure consistency when filtering by +# --do-domain-ids with a start position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="1-1-2"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(2); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-1-2 --do-domain-ids=2 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t3,test.t4' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 16: Ensure consistency when filtering by +# --ignore-domain-ids with a start position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --ignore-domain-ids=0 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 3 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t3 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +connection master; +connection slave; +# t5 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 17: Ensure consistency when filtering by +# --ignore-server-ids with a start position +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(1); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --ignore-server-ids=1 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 2 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +connection master; +connection slave; +# t4 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 18: Ensure consistency when filtering by +# --do-domain-ids with both a start position and stop position that +# all have the same domain id +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0); +START SLAVE UNTIL master_gtid_pos="0-3-3"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --do-domain-ids=0 --stop-position=0-3-3 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 19: Ensure consistency when filtering by +# --do-domain-ids with both a start position and stop position that +# have differing domain ids. Due to the implicit filtering in stop +# position, the result should be empty (no tables replicated). +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(1); +START SLAVE UNTIL master_gtid_pos="0-3-3"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --do-domain-ids=1 --stop-position=0-3-3 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 0 tables +# Verifying integrity of tables.. +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 20: Ensure consistency when filtering by +# --ignore-domain-ids with both a start position and stop position that +# all have the same domain id. The result set should be empty due to +# implicit filtering from stop position and ignoring that same domain. +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0); +START SLAVE UNTIL master_gtid_pos="0-3-3"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --ignore-domain-ids=0 --stop-position=0-3-3 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 0 tables +# Verifying integrity of tables.. +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 21: Ensure consistency when filtering by +# --ignore-domain-ids with both a start position and stop position that +# have differing domain ids. The ignore domain ids should take no +# effect due to the implicit filtering by stop position, i.e. the +# specified domain to ignore is already being filtered. +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-1"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(1); +START SLAVE UNTIL master_gtid_pos="0-3-3"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --ignore-domain-ids=1 --stop-position=0-3-3 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 22: Ensure consistency when filtering by +# --ignore-server-ids with both a start position and stop position. +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=0; +connection slave; +connection master; +# Populating master data +connection slave; +set global gtid_slave_pos="0-1-0"; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(3); +START SLAVE UNTIL master_gtid_pos="0-3-3"; +# No slave error expecting - waiting for slave to catch up to master +# Because there is a stop position we wait for all events to process +# and slave to automatically stop +# Stop slave so it stops receiving master events. +Warnings: +Note 1255 Slave already has been stopped +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --ignore-server-ids=3 --stop-position=0-3-3 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +# All tables are consistent +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +connection master; +# +# +# Test Case 23: Out of order GTIDs from domains or servers which are +# filtered out should not error +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=1; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_SERVER_IDS=(3); +START SLAVE ; +# No slave error expecting - waiting for slave to catch up to master +# Wait for slave IO thread to catch up +# Wait for slave SQL thread to catch up +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t4,test.t5' +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=0 --ignore-server-ids=3 --gtid-strict-mode 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t1 is equivalent on master and slave +# All tables are consistent +# Strict mode enabled - checking mysqlbinlog error output for out +# of order GTIDs +# Not expecting to find out of order GTID error.. +NOT FOUND /Found out of order GTID/ in mysqlbinlog_stderr.out +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t2,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +set @@global.gtid_strict_mode=0; +connection master; +# +# +# Test Case 24: Out of order GTIDs from included domains should error +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=1; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(1); +START SLAVE ; +# slave SQL Thread error expected - waiting for errno 1950 +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t4,test.t5' +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --do-domain-ids=1 --gtid-strict-mode 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t3 is equivalent on master and slave +# All tables are consistent +# Strict mode enabled - checking mysqlbinlog error output for out +# of order GTIDs +# Expecting to find out of order GTID error.. +FOUND 1 /Found out of order GTID/ in mysqlbinlog_stderr.out +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +set @@global.gtid_strict_mode=0; +connection master; +# +# +# Test Case 25: Out of order GTIDs from included servers should error +# +include/mysqlbinlog_slave_consistency.inc +connection slave; +set @@global.gtid_strict_mode=1; +connection slave; +connection master; +# Populating master data +connection slave; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(1); +START SLAVE ; +# slave SQL Thread error expected - waiting for errno 1950 +# Stop slave so it stops receiving master events. +connection master; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t4,test.t5' +RESET MASTER; +# MYSQL_BINLOG BINLOG_FILE_PARAM --ignore-server-ids=1 --gtid-strict-mode 2> MYSQLBINLOG_STDERR | MYSQL +connection slave; +# Checking consistency of 'test' database tables between master and slave +connection master; +# Both servers have 1 tables +# Verifying integrity of tables.. +connection master; +connection slave; +# t2 is equivalent on master and slave +# All tables are consistent +# Strict mode enabled - checking mysqlbinlog error output for out +# of order GTIDs +# Expecting to find out of order GTID error.. +FOUND 1 /Found out of order GTID/ in mysqlbinlog_stderr.out +# Test finished - resetting master and slave.. +connection slave; +RESET SLAVE; +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(), IGNORE_SERVER_IDS=(); +connection master; +RESET MASTER; +DROP TABLE IF EXISTS t1, t2, t3, t4, t5; +Warnings: +Note 1051 Unknown table 'test.t1,test.t3,test.t4,test.t5' +connection slave; +RESET SLAVE; +set global gtid_slave_pos=""; +RESET MASTER; +connection master; +RESET MASTER; +connection slave; +set @@global.gtid_strict_mode=0; +connection master; +# +# +# Test Case 26: Neither mysqlbinlog nor CHANGE MASTER TO should allow +# both do domain ids and ignore domain ids to be set together +# +connection slave; +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_DOMAIN_IDS=(1); +ERROR HY000: Could not initialize master info structure for ''; more error messages can be found in the MariaDB error log +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(); +include/start_slave.inc +connection master; +RESET MASTER; +include/rpl_end.inc +# End of tests (rpl.rpl_mysqlbinlog_slave_consistency) diff --git a/mysql-test/suite/rpl/t/rpl_mysqlbinlog_slave_consistency.test b/mysql-test/suite/rpl/t/rpl_mysqlbinlog_slave_consistency.test new file mode 100644 index 00000000000..d2918e3a890 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mysqlbinlog_slave_consistency.test @@ -0,0 +1,401 @@ +# +# Purpose: +# +# This test ensures the mariadb-binlog command line tool filters events +# by domain id (via --do-domain-ids and --ignore-domain-ids) and server id (via +# --ignore-server-ids) in the same way that a replica server does. +# +# +# Methodology: +# +# This test validates that the domain and server id filtering logic of +# mariadb-binlog matches that of a replica server. In particular, it validates +# a mariadb-binlog replay of a master's binary log is consistent with a +# replica's state which is configured using the same filtering configuration. +# +# It uses a repeatable process to allow for multiple test cases that span +# different filtering configurations. First, a master is seeded with an initial +# set of SQL statements with varying domain and server ids. Then, a set of +# filtering parameters supported by both mariadb-binlog and replica +# capabilities are defined. The replica is configured using these parameters +# and run it until it has processed all events from the primary server; it is +# stopped afterward. For mariadb-binlog validation, the binary log of the +# primary server is copied to a different location for later replay. The +# primary is then reset to its initial state (i.e. the tables are dropped and +# the logs are reset). The mariadb-binlog tool is then used to replay the +# copied binary log file back onto the clean primary server under the same +# filtering conditions as the replica. At this point, the data on the primary +# and replica should be exactly the same because the filtering conditions were +# the same, and all existing tables on both servers are compared using. +# checksums. +# +# +# References: +# +# MDEV-20119: Implement the --do-domain-ids, --ignore-domain-ids, and +# --ignore-server-ids options for mysqlbinlog +# +--source include/master-slave.inc + +--connection slave +SET sql_log_bin=0; +call mtr.add_suppression("Slave: An attempt was made.*"); +call mtr.add_suppression("Both DO_DOMAIN_IDS & IGNORE_DOMAIN_IDS lists can't be non-empty at the same time"); +SET sql_log_bin=1; + +--source include/stop_slave.inc + +--connection master +RESET MASTER; + +# Save old state +let $ORIG_GTID_DOMAIN_ID = `select @@session.gtid_domain_id`; +let $ORIG_SERVER_ID = `select @@session.server_id`; + +--connection slave +--source include/start_slave.inc + +# Initial tests do not use strict mode +--let $strict_mode=0 +--let $slave_sql_errno=0 + +--echo # +--echo # +--echo # Test Case 1: Base case to ensure that mariadb-binlog and replica +--echo # are consistent without any filtering +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 2: Ensure filtering by a single id in --do-domain-ids is +--echo # consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids=0 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 3: Ensure filtering by multiple ids in --do-domain-ids is +--echo # consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids=0,1 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # Test Case 4: Ensure filtering by a single id in --ignore-domain-ids +--echo # is consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=0 +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 5: Ensure filtering by multiple ids in --ignore-domain-ids +--echo # is consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=0,1 +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # Test Case 6: Ensure filtering by a single id in --ignore-server-ids +--echo # is consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=1 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 7: Ensure filtering by multiple ids in --ignore-server-ids +--echo # is consistent between mariadb-binlog and replica +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=0,1 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 8: Ensure stop position consistency +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position=1-1-2 +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 9: Ensure start position consistency +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position=1-4-2 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 10: Ensure consistency when filtering by both +--echo # --do-domain-ids and --ignore-server-ids +--echo # +--let $do_domain_ids=0 +--let $ignore_domain_ids= +--let $ignore_server_ids=1 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 11: Ensure consistency when filtering by both +--echo # --ignore-domain-ids and --ignore-server-ids +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=1,2 +--let $ignore_server_ids=1 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 12: Ensure consistency when filtering by +--echo # --do-domain-ids with a stop position +--echo # +--let $do_domain_ids=0 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position=0-1-4 +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 13: Ensure consistency when filtering by +--echo # --ignore-domain-ids with a stop position +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=0 +--let $ignore_server_ids= +--let $stop_position=0-1-4 +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 14: Ensure consistency when filtering by +--echo # --ignore-server-ids with a stop position +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=3 +--let $stop_position=0-1-4 +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 15: Ensure consistency when filtering by +--echo # --do-domain-ids with a start position +--echo # +--let $do_domain_ids=2 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position=1-1-2 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 16: Ensure consistency when filtering by +--echo # --ignore-domain-ids with a start position +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=0 +--let $ignore_server_ids= +--let $stop_position= +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 17: Ensure consistency when filtering by +--echo # --ignore-server-ids with a start position +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=1 +--let $stop_position= +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 18: Ensure consistency when filtering by +--echo # --do-domain-ids with both a start position and stop position that +--echo # all have the same domain id +--echo # +--let $do_domain_ids=0 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position=0-3-3 +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 19: Ensure consistency when filtering by +--echo # --do-domain-ids with both a start position and stop position that +--echo # have differing domain ids. Due to the implicit filtering in stop +--echo # position, the result should be empty (no tables replicated). +--echo # +--let $do_domain_ids=1 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position=0-3-3 +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 20: Ensure consistency when filtering by +--echo # --ignore-domain-ids with both a start position and stop position that +--echo # all have the same domain id. The result set should be empty due to +--echo # implicit filtering from stop position and ignoring that same domain. +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=0 +--let $ignore_server_ids= +--let $stop_position=0-3-3 +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 21: Ensure consistency when filtering by +--echo # --ignore-domain-ids with both a start position and stop position that +--echo # have differing domain ids. The ignore domain ids should take no +--echo # effect due to the implicit filtering by stop position, i.e. the +--echo # specified domain to ignore is already being filtered. +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids=1 +--let $ignore_server_ids= +--let $stop_position=0-3-3 +--let $start_position=0-1-1 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 22: Ensure consistency when filtering by +--echo # --ignore-server-ids with both a start position and stop position. +--echo # +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=3 +--let $stop_position=0-3-3 +--let $start_position=0-1-0 +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 23: Out of order GTIDs from domains or servers which are +--echo # filtered out should not error +--echo # +--let $strict_mode=1 +--let $strict_mode_err=0 +--let $slave_sql_errno=0 +--let $do_domain_ids=0 +--let $ignore_domain_ids= +--let $ignore_server_ids=3 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 24: Out of order GTIDs from included domains should error +--echo # +--let $strict_mode=1 +--let $strict_mode_err=1 +--let $slave_sql_errno=1950 +--let $do_domain_ids=1 +--let $ignore_domain_ids= +--let $ignore_server_ids= +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 25: Out of order GTIDs from included servers should error +--echo # +--let $strict_mode=1 +--let $strict_mode_err=1 +--let $slave_sql_errno=1950 +--let $do_domain_ids= +--let $ignore_domain_ids= +--let $ignore_server_ids=1 +--let $stop_position= +--let $start_position= +--source include/mysqlbinlog_slave_consistency.inc + +--echo # +--echo # +--echo # Test Case 26: Neither mysqlbinlog nor CHANGE MASTER TO should allow +--echo # both do domain ids and ignore domain ids to be set together +--echo # +--connection slave +--source include/stop_slave.inc + +--error 1201 +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_DOMAIN_IDS=(1); + +--let $MYSQLD_DATADIR=`select @@datadir` +--let BINLOG_FILENAME= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--let BINLOG_FILE_PARAM= $MYSQLD_DATADIR/$BINLOG_FILENAME.orig +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILENAME --do-domain-ids=0 --ignore-domain-ids=1 + +RESET MASTER; +set global gtid_slave_pos=""; +CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(), IGNORE_DOMAIN_IDS=(); +--source include/start_slave.inc + +# Cleanup +--connection master +RESET MASTER; + +--source include/rpl_end.inc + +--echo # End of tests (rpl.rpl_mysqlbinlog_slave_consistency) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index ae40fd46f4d..7876ff456c6 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -16,6 +16,7 @@ /* Definitions for MariaDB global transaction ID (GTID). */ +#include #ifndef MYSQL_CLIENT #include "mariadb.h" @@ -3480,32 +3481,42 @@ my_bool Window_gtid_event_filter::has_finished() return m_has_stop ? m_has_passed : FALSE; } -void free_gtid_filter_element(void *p) +void free_u32_gtid_filter_element(void *p) { - gtid_filter_element *gfe = (gtid_filter_element *) p; + gtid_filter_element *gfe= (gtid_filter_element *) p; if (gfe->filter) delete gfe->filter; my_free(gfe); } -Id_delegating_gtid_event_filter::Id_delegating_gtid_event_filter() - : m_num_stateful_filters(0), m_num_completed_filters(0) +template +Id_delegating_gtid_event_filter::Id_delegating_gtid_event_filter() + : m_num_stateful_filters(0), m_num_completed_filters(0), + m_id_restriction_mode(id_restriction_mode::MODE_NOT_SET) { + void (*free_func)(void *); + if (std::is_same::value) + free_func= free_u32_gtid_filter_element; + else + DBUG_ASSERT(0); + my_hash_init(PSI_INSTRUMENT_ME, &m_filters_by_id_hash, &my_charset_bin, 32, - offsetof(gtid_filter_element, identifier), - sizeof(gtid_filter_identifier), NULL, free_gtid_filter_element, + offsetof(gtid_filter_element, identifier), + sizeof(T), NULL, free_func, HASH_UNIQUE); m_default_filter= new Accept_all_gtid_filter(); } -Id_delegating_gtid_event_filter::~Id_delegating_gtid_event_filter() +template +Id_delegating_gtid_event_filter::~Id_delegating_gtid_event_filter() { my_hash_free(&m_filters_by_id_hash); delete m_default_filter; } -void Id_delegating_gtid_event_filter::set_default_filter( +template +void Id_delegating_gtid_event_filter::set_default_filter( Gtid_event_filter *filter) { if (m_default_filter) @@ -3514,17 +3525,19 @@ void Id_delegating_gtid_event_filter::set_default_filter( m_default_filter= filter; } -gtid_filter_element * -Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( - gtid_filter_identifier filter_id) +template +gtid_filter_element * +Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( + T filter_id) { - gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( - &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + gtid_filter_element *fe= + (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); if (!fe) { - gtid_filter_element *new_fe= (gtid_filter_element *) my_malloc( - PSI_NOT_INSTRUMENTED, sizeof(gtid_filter_element), MYF(MY_WME)); + gtid_filter_element *new_fe= (gtid_filter_element *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(gtid_filter_element), MYF(MY_WME)); new_fe->filter= NULL; new_fe->identifier= filter_id; if (my_hash_insert(&m_filters_by_id_hash, (uchar*) new_fe)) @@ -3539,7 +3552,8 @@ Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( return fe; } -my_bool Id_delegating_gtid_event_filter::has_finished() +template +my_bool Id_delegating_gtid_event_filter::has_finished() { /* If all user-defined filters have deactivated, we are effectively @@ -3549,11 +3563,13 @@ my_bool Id_delegating_gtid_event_filter::has_finished() m_num_completed_filters == m_num_stateful_filters; } -my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) +template +my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) { - gtid_filter_identifier filter_id= get_id_from_gtid(gtid); - gtid_filter_element *filter_element= (gtid_filter_element *) my_hash_search( - &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + T filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= + (gtid_filter_element *) my_hash_search(&m_filters_by_id_hash, + (const uchar *) &filter_id, 0); Gtid_event_filter *filter= (filter_element ? filter_element->filter : m_default_filter); my_bool ret= TRUE; @@ -3573,11 +3589,124 @@ my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) return ret; } + +template Gtid_event_filter* create_event_filter() +{ + return new F(); +} + +template +int Id_delegating_gtid_event_filter::set_id_restrictions( + T *id_list, size_t n_ids, id_restriction_mode mode) +{ + static const char *WHITELIST_NAME= "do", *BLACKLIST_NAME= "ignore"; + + size_t id_ctr; + int err; + Gtid_event_filter::gtid_event_filter_type filter_type; + const char *filter_name, *opposite_filter_name; + Gtid_event_filter *(*construct_filter)(void); + Gtid_event_filter *(*construct_default_filter)(void); + + DBUG_ASSERT(mode > id_restriction_mode::MODE_NOT_SET); + + /* + Set up variables which help this filter either be in whitelist or blacklist + mode + */ + if (mode == Gtid_event_filter::id_restriction_mode::WHITELIST_MODE) + { + filter_type= Gtid_event_filter::ACCEPT_ALL_GTID_FILTER_TYPE; + filter_name= WHITELIST_NAME; + opposite_filter_name= BLACKLIST_NAME; + construct_filter= + create_event_filter; + construct_default_filter= + create_event_filter; + } + else if (mode == Gtid_event_filter::id_restriction_mode::BLACKLIST_MODE) + { + filter_type= Gtid_event_filter::REJECT_ALL_GTID_FILTER_TYPE; + filter_name= BLACKLIST_NAME; + opposite_filter_name= WHITELIST_NAME; + construct_filter= + create_event_filter; + construct_default_filter= + create_event_filter; + } + else + { + DBUG_ASSERT(0); + } + + if (m_id_restriction_mode != + Gtid_event_filter::id_restriction_mode::MODE_NOT_SET) + { + if (mode != m_id_restriction_mode) + { + /* + If a rule specifying the opposite version of this has already been set, + error. + */ + sql_print_error("Cannot create %s filtering rule for %s id because " + "%s rule already exists", + filter_name, get_id_type_name(), + opposite_filter_name); + err= 1; + goto err; + } + + /* This filter is specified more than once, only use the latest values */ + my_hash_reset(&m_filters_by_id_hash); + } + + for (id_ctr= 0; id_ctr < n_ids; id_ctr++) + { + T filter_id= id_list[id_ctr]; + gtid_filter_element *map_element= + find_or_create_filter_element_for_id(filter_id); + + if(map_element == NULL) + { + /* + If map_element is NULL, find_or_create_filter_element_for_id failed and + has already written the error message + */ + err= 1; + goto err; + } + else if (map_element->filter == NULL) + { + map_element->filter= construct_filter(); + m_num_stateful_filters++; + } + else + { + DBUG_ASSERT(map_element->filter->get_filter_type() == + filter_type); + } + } + + /* + With a whitelist, we by only want to accept the ids which are specified. + Everything else should be denied. + + With a blacklist, we by default want to accept everything that is not + specified in the list + */ + set_default_filter(construct_default_filter()); + m_id_restriction_mode= mode; + err= 0; + +err: + return err; +} + Window_gtid_event_filter * Domain_gtid_event_filter::find_or_create_window_filter_for_id( - uint32 domain_id) + decltype(rpl_gtid::domain_id) domain_id) { - gtid_filter_element *filter_element= + gtid_filter_element *filter_element= find_or_create_filter_element_for_id(domain_id); Window_gtid_event_filter *wgef= NULL; @@ -3606,9 +3735,11 @@ Domain_gtid_event_filter::find_or_create_window_filter_for_id( return wgef; } -static my_bool check_filter_entry_validity(void *entry, void *are_filters_invalid_arg) +static my_bool check_filter_entry_validity(void *entry, + void *are_filters_invalid_arg) { - gtid_filter_element *fe= (gtid_filter_element*) entry; + gtid_filter_element *fe= + (gtid_filter_element *) entry; if (fe) { @@ -3629,7 +3760,8 @@ static my_bool check_filter_entry_validity(void *entry, void *are_filters_invali int Domain_gtid_event_filter::validate_window_filters() { int are_filters_invalid= 0; - my_hash_iterate(&m_filters_by_id_hash, check_filter_entry_validity, &are_filters_invalid); + my_hash_iterate(&m_filters_by_id_hash, check_filter_entry_validity, + &are_filters_invalid); return are_filters_invalid; } @@ -3645,8 +3777,9 @@ int Domain_gtid_event_filter::add_start_gtid(rpl_gtid *gtid) } else if (!(err= filter_to_update->set_start_gtid(gtid))) { - gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( - &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + gtid_filter_element *fe= + (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); insert_dynamic(&m_start_filters, (const void *) &fe); } @@ -3665,8 +3798,9 @@ int Domain_gtid_event_filter::add_stop_gtid(rpl_gtid *gtid) } else if (!(err= filter_to_update->set_stop_gtid(gtid))) { - gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( - &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + gtid_filter_element *fe= + (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); insert_dynamic(&m_stop_filters, (const void *) &fe); /* @@ -3699,8 +3833,9 @@ rpl_gtid *Domain_gtid_event_filter::get_start_gtids() for (i = 0; i < n_start_gtids; i++) { - gtid_filter_element *fe= - *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + gtid_filter_element *fe= + *(gtid_filter_element **) + dynamic_array_ptr(&m_start_filters, i); DBUG_ASSERT(fe->filter && fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); Window_gtid_event_filter *wgef= @@ -3724,8 +3859,9 @@ rpl_gtid *Domain_gtid_event_filter::get_stop_gtids() for (i = 0; i < n_stop_gtids; i++) { - gtid_filter_element *fe= - *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + gtid_filter_element *fe= + *(gtid_filter_element **) + dynamic_array_ptr(&m_stop_filters, i); DBUG_ASSERT(fe->filter && fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); Window_gtid_event_filter *wgef= @@ -3743,8 +3879,9 @@ void Domain_gtid_event_filter::clear_start_gtids() uint32 i; for (i = 0; i < get_num_start_gtids(); i++) { - gtid_filter_element *fe= - *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + gtid_filter_element *fe= + *(gtid_filter_element **) + dynamic_array_ptr(&m_start_filters, i); DBUG_ASSERT(fe->filter && fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); Window_gtid_event_filter *wgef= @@ -3775,8 +3912,9 @@ void Domain_gtid_event_filter::clear_stop_gtids() for (i = 0; i < get_num_stop_gtids(); i++) { - gtid_filter_element *fe= - *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + gtid_filter_element *fe= + *(gtid_filter_element **) + dynamic_array_ptr(&m_stop_filters, i); DBUG_ASSERT(fe->filter && fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); Window_gtid_event_filter *wgef= @@ -3822,10 +3960,10 @@ my_bool Domain_gtid_event_filter::exclude(rpl_gtid *gtid) */ if (get_num_stop_gtids()) { - gtid_filter_identifier filter_id= get_id_from_gtid(gtid); - gtid_filter_element *filter_element= - (gtid_filter_element *) my_hash_search(&m_filters_by_id_hash, - (const uchar *) &filter_id, 0); + decltype(rpl_gtid::domain_id) filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= + (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); if (filter_element) { Gtid_event_filter *filter= filter_element->filter; @@ -3840,3 +3978,52 @@ my_bool Domain_gtid_event_filter::exclude(rpl_gtid *gtid) return include_domain ? Id_delegating_gtid_event_filter::exclude(gtid) : TRUE; } + +Intersecting_gtid_event_filter::Intersecting_gtid_event_filter( + Gtid_event_filter *filter1, Gtid_event_filter *filter2) +{ + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_filters, + sizeof(Gtid_event_filter *), 3, 3, MYF(0)); + insert_dynamic(&m_filters, (void *) &filter1); + insert_dynamic(&m_filters, (void *) &filter2); +} + +Intersecting_gtid_event_filter::~Intersecting_gtid_event_filter() +{ + Gtid_event_filter *tmp_filter= NULL; + ulong i; + for (i= 0; i < m_filters.elements; i++) + { + tmp_filter= *(Gtid_event_filter **) dynamic_array_ptr(&m_filters, i); + delete tmp_filter; + } + delete_dynamic(&m_filters); +} + +my_bool Intersecting_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + Gtid_event_filter *tmp_filter= NULL; + ulong i; + for (i= 0; i < m_filters.elements; i++) + { + tmp_filter= *(Gtid_event_filter **) dynamic_array_ptr(&m_filters, i); + DBUG_ASSERT(tmp_filter); + if (tmp_filter->exclude(gtid)) + return TRUE; + } + return FALSE; +} + +my_bool Intersecting_gtid_event_filter::has_finished() +{ + Gtid_event_filter *tmp_filter= NULL; + ulong i; + for (i= 0; i < m_filters.elements; i++) + { + tmp_filter= *(Gtid_event_filter **) dynamic_array_ptr(&m_filters, i); + DBUG_ASSERT(tmp_filter); + if (tmp_filter->has_finished()) + return TRUE; + } + return FALSE; +} diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index 759e4e4eea3..8bbb87c0e81 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -37,9 +37,6 @@ struct rpl_gtid uint64 seq_no; }; -/* Data structure to help with quick lookup for filters. */ -typedef decltype(rpl_gtid::domain_id) gtid_filter_identifier; - inline bool operator==(const rpl_gtid& lhs, const rpl_gtid& rhs) { return @@ -464,7 +461,8 @@ public: Ensures that the expected stop GTID positions exist within the specified binary logs. */ - my_bool verify_stop_state(FILE *out, rpl_gtid *stop_gtids, size_t n_stop_gtids); + my_bool verify_stop_state(FILE *out, rpl_gtid *stop_gtids, + size_t n_stop_gtids); /* Ensure a GTID state (e.g., from a Gtid_list_log_event) is consistent with @@ -513,8 +511,8 @@ public: private: /* - Holds the records for each domain id we are monitoring. Elements are of type - `struct audit_elem` and indexed by domian_id. + Holds the records for each domain id we are monitoring. Elements are of + type `struct audit_elem` and indexed by domian_id. */ HASH m_audit_elem_domain_lookup; }; @@ -533,7 +531,15 @@ public: DELEGATING_GTID_FILTER_TYPE = 1, WINDOW_GTID_FILTER_TYPE = 2, ACCEPT_ALL_GTID_FILTER_TYPE = 3, - REJECT_ALL_GTID_FILTER_TYPE = 4 + REJECT_ALL_GTID_FILTER_TYPE = 4, + INTERSECTING_GTID_FILTER_TYPE = 5 + }; + + enum class id_restriction_mode + { + MODE_NOT_SET, + WHITELIST_MODE, + BLACKLIST_MODE, }; /* @@ -596,8 +602,9 @@ public: positions, m_start (exclusive) and m_stop (inclusive), within a domain. This filter is stateful, such that it expects GTIDs to be an increasing - stream, and internally, the window will activate and deactivate when the start - and stop positions of the event stream have passed through, respectively. + stream, and internally, the window will activate and deactivate when the + start and stop positions of the event stream have passed through, + respectively. */ class Window_gtid_event_filter : public Gtid_event_filter { @@ -701,11 +708,11 @@ private: rpl_gtid m_stop; }; -typedef struct _gtid_filter_element +template struct gtid_filter_element { Gtid_event_filter *filter; - gtid_filter_identifier identifier; /* Used for HASH lookup */ -} gtid_filter_element; + T identifier; /* Used for HASH lookup */ +}; /* Gtid_event_filter subclass which has no specific implementation, but rather @@ -715,8 +722,10 @@ typedef struct _gtid_filter_element filter can be identified. This class should be subclassed, where the get_id_from_gtid function - specifies how to extract the filter identifier from a GTID. + specifies how to extract the filter identifier from a GTID. The type of the + filter identifier is a template for the class. */ +template class Id_delegating_gtid_event_filter : public Gtid_event_filter { public: @@ -729,7 +738,21 @@ public: uint32 get_filter_type() { return DELEGATING_GTID_FILTER_TYPE; } - virtual gtid_filter_identifier get_id_from_gtid(rpl_gtid *) = 0; + virtual T get_id_from_gtid(rpl_gtid *) = 0; + virtual const char* get_id_type_name() = 0; + + /* + Sets restrictions on entire ids using the corresponding mode (i.e. either + whitelist or blacklist, refer to Gtid_event_filter::id_restriction_mode) + + A blacklist will allow all ids except for the ones provided in the input + list. + A whitelist will only allow ids provided in the input list. + + Returns 0 on ok, non-zero on error. + */ + int set_id_restrictions(T *id_list, size_t n_ids, + Gtid_event_filter::id_restriction_mode mode); protected: @@ -739,12 +762,14 @@ protected: HASH m_filters_by_id_hash; - gtid_filter_element *find_or_create_filter_element_for_id(gtid_filter_identifier); + Gtid_event_filter::id_restriction_mode m_id_restriction_mode; + + gtid_filter_element *find_or_create_filter_element_for_id(T); }; /* - A subclass of Id_delegating_gtid_event_filter which identifies filters using the - domain id of a GTID. + A subclass of Id_delegating_gtid_event_filter which identifies filters using + the domain id of a GTID. Additional helper functions include: add_start_gtid(GTID) : adds a start GTID position to this filter, to be @@ -758,15 +783,18 @@ protected: get_num_start_gtids() : gets the count of added GTID start positions get_num_stop_gtids() : gets the count of added GTID stop positions */ -class Domain_gtid_event_filter : public Id_delegating_gtid_event_filter +class Domain_gtid_event_filter + : public Id_delegating_gtid_event_filter { public: Domain_gtid_event_filter() { my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_start_filters, - sizeof(gtid_filter_element*), 8, 8, MYF(0)); + sizeof(decltype(rpl_gtid::domain_id) *), 8, 8, + MYF(0)); my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_stop_filters, - sizeof(gtid_filter_element*), 8, 8, MYF(0)); + sizeof(decltype(rpl_gtid::domain_id) *), 8, 8, + MYF(0)); } ~Domain_gtid_event_filter() { @@ -777,11 +805,13 @@ public: /* Returns the domain id of from the input GTID */ - gtid_filter_identifier get_id_from_gtid(rpl_gtid *gtid) + decltype(rpl_gtid::domain_id) get_id_from_gtid(rpl_gtid *gtid) { return gtid->domain_id; } + const char* get_id_type_name() { return "domain"; } + /* Override Id_delegating_gtid_event_filter to extend with domain specific filtering logic @@ -838,7 +868,65 @@ private: DYNAMIC_ARRAY m_start_filters; DYNAMIC_ARRAY m_stop_filters; - Window_gtid_event_filter *find_or_create_window_filter_for_id(gtid_filter_identifier); + Window_gtid_event_filter * + find_or_create_window_filter_for_id(decltype(rpl_gtid::domain_id)); +}; + +/* + A subclass of Id_delegating_gtid_event_filter which identifies filters using + the server id of a GTID. +*/ +class Server_gtid_event_filter + : public Id_delegating_gtid_event_filter +{ +public: + /* + Returns the server id of from the input GTID + */ + decltype(rpl_gtid::server_id) get_id_from_gtid(rpl_gtid *gtid) + { + return gtid->server_id; + } + + const char* get_id_type_name() { return "server"; } +}; + +/* + A Gtid_event_filter implementation that delegates the filtering to other + filters, where the result is the intersection between them all. +*/ +class Intersecting_gtid_event_filter : public Gtid_event_filter +{ +public: + Intersecting_gtid_event_filter(Gtid_event_filter *filter1, + Gtid_event_filter *filter2); + ~Intersecting_gtid_event_filter(); + + /* + Returns TRUE if any filers exclude the gtid, returns FALSE otherwise, i.e. + all filters must allow the GTID. + */ + my_bool exclude(rpl_gtid *gtid); + + /* + Returns true if any filters have finished. To elaborate, as this filter + performs an intersection, if any filter has finished, the result would + be excluded regardless. + */ + my_bool has_finished(); + + uint32 get_filter_type() { return INTERSECTING_GTID_FILTER_TYPE; } + + /* + Adds a new filter to the intersection + */ + my_bool add_filter(Gtid_event_filter *filter) + { + return insert_dynamic(&m_filters, (void *) &filter); + } + + protected: + DYNAMIC_ARRAY m_filters; }; #endif /* RPL_GTID_H */