MDEV-39958 Galera support for --binlog-storage-engine

phase 3 of the development: state transfers and more tests

Make SST work with the binlog in InnoDB, both when all nodes
use it and in a cluster where some nodes use a file binlog.

Several new mtr tests.

Known issues: joiner using  binlog-in-engine binlog receiving SST
from a donor using file binlog has no binlog GTID position in the
wsrep domain until it binlogs a transaction.
joiner using binlog files receiving  rsync SST from a donor using
binlog-in-engine, will keep the copied .ibb files.

extra/mariabackup/backup_mysql.cc

@@ -1610,6 +1610,7 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection)
 	char *log_bin_pos = NULL;
 	char *log_bin_gtid_executed = NULL;
 	char *log_bin_dir = NULL;
+	char *binlog_storage_engine = NULL;
 	bool gtid_exists;
 	bool result = true;
 	char filepath[FN_REFLEN];
@@ -1632,12 +1633,21 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection)
 		{"gtid_binlog_state", &gtid_binlog_state},
 		{"gtid_current_pos", &gtid_current_pos},
 		{"log_bin_basename", &log_bin_dir},
+		{"binlog_storage_engine", &binlog_storage_engine},
 		{NULL, NULL}
 	};
 
 	read_mysql_variables(connection, "SHOW MASTER STATUS", status, false);
 	read_mysql_variables(connection, "SHOW VARIABLES", vars, true);
 
+	/*
+	  With --binlog-storage-engine, the binlog files are copied by
+	  copy_engine_binlogs() consistently with the engine data, so they
+	  are not flushed and copied here.
+	*/
+	if (binlog_storage_engine && *binlog_storage_engine)
+		goto cleanup;
+
 	gtid_exists = (executed_gtid_set && *executed_gtid_set)
 			|| (gtid_binlog_state && *gtid_binlog_state);
 

mysql-test/suite/galera/include/galera_binlog_in_engine_check_gtid.inc

@@ -0,0 +1,18 @@
+# Check that node_1 and node_2 have binlogged the same last replicated
+# transaction, in wsrep_gtid_domain_id (1). Both nodes must be synced.
+# See also galera_binlog_in_engine_check_gtid_history.inc.
+--connection node_1
+--let $gtid_1 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(?<![0-9])1-[0-9]+-[0-9]+')`
+--connection node_2
+--let $wait_condition = SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(?<![0-9])1-[0-9]+-[0-9]+') = '$gtid_1'
+--source include/wait_condition.inc
+--let $gtid_2 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(?<![0-9])1-[0-9]+-[0-9]+')`
+if (!$gtid_1)
+{
+  --die No replicated transactions in gtid_binlog_pos of node_1
+}
+if ($gtid_1 != $gtid_2)
+{
+  --die gtid_binlog_pos domain 1 differs between the nodes: $gtid_1 vs $gtid_2
+}
+--echo # gtid_binlog_pos domain 1 matches on both nodes

mysql-test/suite/galera/include/galera_binlog_in_engine_check_gtid_history.inc

@@ -0,0 +1,21 @@
+# Check that node_1 and node_2 have binlogged the same replicated
+# transactions, in wsrep_gtid_domain_id (1): the last GTID, and the whole
+# sequence of GTIDs read from the binlog of each node with mariadb-binlog.
+# (Not for a node which uses a file binlog: it starts a fresh binlog after a
+# mariabackup or rsync SST, without the history from before the SST.)
+# Both nodes must be synced.
+--source suite/galera/include/galera_binlog_in_engine_check_gtid.inc
+--perl
+  my @seq;
+  for my $port ($ENV{NODE_MYPORT_1}, $ENV{NODE_MYPORT_2}) {
+    # Read from the start of the binlog: a GTID position in an unused
+    # domain, with non-strict mode.
+    my $out= `$ENV{MYSQL_BINLOG} --read-from-remote-server --user=root --host=127.0.0.1 --port=$port --start-position=42-42-42 --gtid-strict-mode=0 master-bin.000001 2>&1`;
+    die "mariadb-binlog failed for port $port: $out" if $?;
+    push @seq, join(' ', $out =~ /GTID (1-\d+-\d+)/g);
+  }
+  die "No replicated transactions in the binlog\n" unless $seq[0];
+  die "The binlogs differ:\nnode_1: $seq[0]\nnode_2: $seq[1]\n"
+    if $seq[0] ne $seq[1];
+  print "# The binlogs of both nodes have the same replicated GTIDs\n";
+EOF

mysql-test/suite/galera/include/galera_binlog_in_engine_mixed.inc

@@ -0,0 +1,26 @@
+# Workload for a cluster where one node uses the binlog in InnoDB
+# (--binlog-storage-engine=innodb) and the other a file binlog.
+# Transactions small and larger than the binlog cache, and streaming
+# replication, are run from both nodes, before and after a state transfer
+# to node_2: an SST to a clean data directory, and an IST after a kill.
+
+--connection node_1
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+--connection node_2
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+# (No while loop: mysqltest does not run a perl block, as in
+# galera_wsrep_recover.inc, again in a later iteration.)
+--source suite/galera/include/galera_binlog_in_engine_mixed_round.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+--source suite/galera/include/galera_binlog_in_engine_mixed_round.inc
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_binlog_in_engine_mixed_round.inc
+
+--source include/auto_increment_offset_restore.inc

mysql-test/suite/galera/include/galera_binlog_in_engine_mixed_round.inc

@@ -0,0 +1,46 @@
+# One round of the workload of galera_binlog_in_engine_mixed.inc.
+
+# galera_st_clean_slave.inc and galera_st_kill_slave.inc leave autocommit
+# off in the node_2 session.
+--connection node_1
+SET SESSION autocommit = 1;
+--connection node_2
+SET SESSION autocommit = 1;
+
+--connection node_1
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 101 FROM t2 WHERE b LIKE 's%'
+--source include/wait_condition.inc
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+
+--connection node_1
+--let $wait_condition = SELECT COUNT(*) = 101 FROM t2 WHERE b LIKE 't%'
+--source include/wait_condition.inc
+--let $data_1 = `SELECT CONCAT(COUNT(*), ' ', SUM(CRC32(b))) FROM t2`
+--connection node_2
+--let $data_2 = `SELECT CONCAT(COUNT(*), ' ', SUM(CRC32(b))) FROM t2`
+if ($data_1 != $data_2)
+{
+  --die The nodes have different data in t2: node_1 $data_1, node_2 $data_2
+}
+--echo # t2 has the same data on both nodes: $data_1
+
+--connection node_1
+DROP TABLE t2;
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 0 FROM information_schema.tables WHERE table_name = 't2'
+--source include/wait_condition.inc
+--source suite/galera/include/galera_binlog_in_engine_check_gtid.inc

mysql-test/suite/galera/r/galera_binlog_in_engine.result

@@ -164,4 +164,6 @@ DROP TABLE t1;
 #
 connection node_2;
 connection node_1;
+connection node_2;
 # gtid_binlog_pos domain 1 matches on both nodes
+# The binlogs of both nodes have the same replicated GTIDs

mysql-test/suite/galera/r/galera_binlog_in_engine_mixed_engine_donor.result

@@ -0,0 +1,444 @@
+connection node_2;
+connection node_1;
+connection node_1;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+@@log_bin	@@binlog_storage_engine
+1	innodb
+connection node_2;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+@@log_bin	@@binlog_storage_engine
+1	NULL
+connection node_1;
+connection node_2;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+Performing State Transfer on a server that starts from a clean var directory
+This is accomplished by shutting down node #2 and removing its var directory before restarting it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+Cleaning var directory ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_clean_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/r/galera_binlog_in_engine_mixed_file_donor.result

@@ -0,0 +1,444 @@
+connection node_2;
+connection node_1;
+connection node_1;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+@@log_bin	@@binlog_storage_engine
+1	NULL
+connection node_2;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@log_bin, @@binlog_storage_engine;
+@@log_bin	@@binlog_storage_engine
+1	innodb
+connection node_1;
+connection node_2;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+Performing State Transfer on a server that starts from a clean var directory
+This is accomplished by shutting down node #2 and removing its var directory before restarting it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+Cleaning var directory ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_clean_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+connection node_1;
+SET SESSION autocommit = 1;
+connection node_2;
+SET SESSION autocommit = 1;
+connection node_1;
+CREATE TABLE t2 (id INT PRIMARY KEY, node INT, b LONGBLOB) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1, 1, 'small');
+INSERT INTO t2 SELECT seq, 1, REPEAT('x', 2000) FROM seq_100_to_199;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('s', 3000) WHERE node = 1;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_2;
+INSERT INTO t2 VALUES (2, 2, 'small');
+INSERT INTO t2 SELECT seq, 2, REPEAT('y', 2000) FROM seq_200_to_299;
+SET SESSION wsrep_trx_fragment_unit = 'bytes';
+SET SESSION wsrep_trx_fragment_size = 20000;
+UPDATE t2 SET b = REPEAT('t', 3000) WHERE node = 2;
+SET SESSION wsrep_trx_fragment_size = 0;
+connection node_1;
+connection node_2;
+# t2 has the same data on both nodes: 202 766646630397
+connection node_1;
+DROP TABLE t2;
+connection node_2;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/r/galera_binlog_in_engine_sst_mariabackup.result

@@ -0,0 +1,732 @@
+connection node_2;
+connection node_1;
+connection node_1;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+connection node_2;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+connection node_1;
+connection node_2;
+connection node_1;
+select @@innodb_undo_tablespaces;
+@@innodb_undo_tablespaces
+0
+connection node_2;
+select @@innodb_undo_tablespaces;
+@@innodb_undo_tablespaces
+3
+Performing State Transfer on a server that has been shut down cleanly and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_shutdown_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that starts from a clean var directory
+This is accomplished by shutting down node #2 and removing its var directory before restarting it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+Cleaning var directory ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_clean_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been killed and restarted
+while a DDL was in progress on it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+connection node_2;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
+connection node_1;
+ALTER TABLE t1 ADD COLUMN f2 INTEGER;
+connection node_2;
+SET wsrep_sync_wait = 0;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+connection node_2;
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave_ddl;
+INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+EXPECT_3
+3
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1	f2
+1	node1_committed_before	NULL
+2	node1_committed_before	NULL
+3	node1_committed_before	NULL
+4	node1_committed_before	NULL
+5	node1_committed_before	NULL
+6	node2_committed_before	NULL
+7	node2_committed_before	NULL
+8	node2_committed_before	NULL
+9	node2_committed_before	NULL
+10	node2_committed_before	NULL
+11	node1_committed_during	NULL
+12	node1_committed_during	NULL
+13	node1_committed_during	NULL
+14	node1_committed_during	NULL
+15	node1_committed_during	NULL
+16	node1_to_be_committed_after	NULL
+17	node1_to_be_committed_after	NULL
+18	node1_to_be_committed_after	NULL
+19	node1_to_be_committed_after	NULL
+20	node1_to_be_committed_after	NULL
+26	node2_committed_after	NULL
+27	node2_committed_after	NULL
+28	node2_committed_after	NULL
+29	node2_committed_after	NULL
+30	node2_committed_after	NULL
+31	node1_to_be_committed_after	NULL
+32	node1_to_be_committed_after	NULL
+33	node1_to_be_committed_after	NULL
+34	node1_to_be_committed_after	NULL
+35	node1_to_be_committed_after	NULL
+36	node1_committed_after	NULL
+37	node1_committed_after	NULL
+38	node1_committed_after	NULL
+39	node1_committed_after	NULL
+40	node1_committed_after	NULL
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+EXPECT_3
+3
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1	f2
+1	node1_committed_before	NULL
+2	node1_committed_before	NULL
+3	node1_committed_before	NULL
+4	node1_committed_before	NULL
+5	node1_committed_before	NULL
+6	node2_committed_before	NULL
+7	node2_committed_before	NULL
+8	node2_committed_before	NULL
+9	node2_committed_before	NULL
+10	node2_committed_before	NULL
+11	node1_committed_during	NULL
+12	node1_committed_during	NULL
+13	node1_committed_during	NULL
+14	node1_committed_during	NULL
+15	node1_committed_during	NULL
+16	node1_to_be_committed_after	NULL
+17	node1_to_be_committed_after	NULL
+18	node1_to_be_committed_after	NULL
+19	node1_to_be_committed_after	NULL
+20	node1_to_be_committed_after	NULL
+26	node2_committed_after	NULL
+27	node2_committed_after	NULL
+28	node2_committed_after	NULL
+29	node2_committed_after	NULL
+30	node2_committed_after	NULL
+31	node1_to_be_committed_after	NULL
+32	node1_to_be_committed_after	NULL
+33	node1_to_be_committed_after	NULL
+34	node1_to_be_committed_after	NULL
+35	node1_to_be_committed_after	NULL
+36	node1_committed_after	NULL
+37	node1_committed_after	NULL
+38	node1_committed_after	NULL
+39	node1_committed_after	NULL
+40	node1_committed_after	NULL
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET GLOBAL debug_dbug = $debug_orig;
+connection node_2;
+Shutting down server ...
+connection node_1;
+connection node_2;
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+connection node_1;
+connection node_2;
+select @@innodb_undo_tablespaces;
+@@innodb_undo_tablespaces
+3
+call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=3 because previous shutdown was not with innodb_fast_shutdown=0");
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+# The binlogs of both nodes have the same replicated GTIDs
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/r/galera_binlog_in_engine_sst_rsync.result

@@ -0,0 +1,710 @@
+connection node_2;
+connection node_1;
+connection node_1;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+connection node_2;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+connection node_1;
+connection node_2;
+Performing State Transfer on a server that has been shut down cleanly and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_shutdown_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that starts from a clean var directory
+This is accomplished by shutting down node #2 and removing its var directory before restarting it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+Cleaning var directory ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_clean_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been killed and restarted
+while a DDL was in progress on it
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+connection node_2;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
+connection node_1;
+ALTER TABLE t1 ADD COLUMN f2 INTEGER;
+connection node_2;
+SET wsrep_sync_wait = 0;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during');
+INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+connection node_2;
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after');
+INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after');
+INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave_ddl;
+INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+EXPECT_3
+3
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1	f2
+1	node1_committed_before	NULL
+2	node1_committed_before	NULL
+3	node1_committed_before	NULL
+4	node1_committed_before	NULL
+5	node1_committed_before	NULL
+6	node2_committed_before	NULL
+7	node2_committed_before	NULL
+8	node2_committed_before	NULL
+9	node2_committed_before	NULL
+10	node2_committed_before	NULL
+11	node1_committed_during	NULL
+12	node1_committed_during	NULL
+13	node1_committed_during	NULL
+14	node1_committed_during	NULL
+15	node1_committed_during	NULL
+16	node1_to_be_committed_after	NULL
+17	node1_to_be_committed_after	NULL
+18	node1_to_be_committed_after	NULL
+19	node1_to_be_committed_after	NULL
+20	node1_to_be_committed_after	NULL
+26	node2_committed_after	NULL
+27	node2_committed_after	NULL
+28	node2_committed_after	NULL
+29	node2_committed_after	NULL
+30	node2_committed_after	NULL
+31	node1_to_be_committed_after	NULL
+32	node1_to_be_committed_after	NULL
+33	node1_to_be_committed_after	NULL
+34	node1_to_be_committed_after	NULL
+35	node1_to_be_committed_after	NULL
+36	node1_committed_after	NULL
+37	node1_committed_after	NULL
+38	node1_committed_after	NULL
+39	node1_committed_after	NULL
+40	node1_committed_after	NULL
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+EXPECT_3
+3
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1	f2
+1	node1_committed_before	NULL
+2	node1_committed_before	NULL
+3	node1_committed_before	NULL
+4	node1_committed_before	NULL
+5	node1_committed_before	NULL
+6	node2_committed_before	NULL
+7	node2_committed_before	NULL
+8	node2_committed_before	NULL
+9	node2_committed_before	NULL
+10	node2_committed_before	NULL
+11	node1_committed_during	NULL
+12	node1_committed_during	NULL
+13	node1_committed_during	NULL
+14	node1_committed_during	NULL
+15	node1_committed_during	NULL
+16	node1_to_be_committed_after	NULL
+17	node1_to_be_committed_after	NULL
+18	node1_to_be_committed_after	NULL
+19	node1_to_be_committed_after	NULL
+20	node1_to_be_committed_after	NULL
+26	node2_committed_after	NULL
+27	node2_committed_after	NULL
+28	node2_committed_after	NULL
+29	node2_committed_after	NULL
+30	node2_committed_after	NULL
+31	node1_to_be_committed_after	NULL
+32	node1_to_be_committed_after	NULL
+33	node1_to_be_committed_after	NULL
+34	node1_to_be_committed_after	NULL
+35	node1_to_be_committed_after	NULL
+36	node1_committed_after	NULL
+37	node1_committed_after	NULL
+38	node1_committed_after	NULL
+39	node1_committed_after	NULL
+40	node1_committed_after	NULL
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET GLOBAL debug_dbug = $debug_orig;
+connection node_1;
+connection node_2;
+# gtid_binlog_pos domain 1 matches on both nodes
+# The binlogs of both nodes have the same replicated GTIDs

mysql-test/suite/galera/t/galera_binlog_in_engine.test

@@ -212,15 +212,4 @@ DROP TABLE t1;
 --connection node_2
 --let $wait_condition = SELECT COUNT(*) = 0 FROM information_schema.tables WHERE table_name = 't1'
 --source include/wait_condition.inc
---let $gtid_2 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(^|,)1-[0-9]+-[0-9]+')`
---connection node_1
---let $gtid_1 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(^|,)1-[0-9]+-[0-9]+')`
-if (!$gtid_1)
-{
-  --die No replicated transactions in gtid_binlog_pos of node_1
-}
-if ($gtid_1 != $gtid_2)
-{
-  --die gtid_binlog_pos domain 1 differs between the nodes: $gtid_1 vs $gtid_2
-}
---echo # gtid_binlog_pos domain 1 matches on both nodes
+--source suite/galera/include/galera_binlog_in_engine_check_gtid_history.inc

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_engine_donor.cnf

@@ -0,0 +1,23 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+log-bin
+log-slave-updates
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+binlog-cache-size=32768
+
+[mysqld.1]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+server-id=11
+
+[mysqld.2]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+server-id=12
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream
+
+[mysqld.1]
+binlog-storage-engine=innodb

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_engine_donor.combinations

@@ -0,0 +1,6 @@
+[rsync]
+wsrep_sst_method=rsync
+
+[mariabackup]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_engine_donor.test

@@ -0,0 +1,14 @@
+#
+# A cluster where one node uses the binlog in InnoDB
+# (--binlog-storage-engine=innodb) and the other a file binlog, see
+# the .cnf file. Here node_1, the donor for the state
+# transfers to node_2, uses the binlog in InnoDB.
+#
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_mariabackup.inc
+
+--source suite/galera/include/galera_binlog_in_engine_mixed.inc
+
+--source include/galera_end.inc

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_file_donor.cnf

@@ -0,0 +1,23 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+log-bin
+log-slave-updates
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+binlog-cache-size=32768
+
+[mysqld.1]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+server-id=11
+
+[mysqld.2]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+server-id=12
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream
+
+[mysqld.2]
+binlog-storage-engine=innodb

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_file_donor.combinations

@@ -0,0 +1,6 @@
+[rsync]
+wsrep_sst_method=rsync
+
+[mariabackup]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:

mysql-test/suite/galera/t/galera_binlog_in_engine_mixed_file_donor.test

@@ -0,0 +1,14 @@
+#
+# A cluster where one node uses the binlog in InnoDB
+# (--binlog-storage-engine=innodb) and the other a file binlog, see
+# the .cnf file. Here node_1, the donor for the state
+# transfers to node_2, uses a file binlog.
+#
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_mariabackup.inc
+
+--source suite/galera/include/galera_binlog_in_engine_mixed.inc
+
+--source include/galera_end.inc

mysql-test/suite/galera/t/galera_binlog_in_engine_sst_mariabackup.cnf

@@ -0,0 +1,37 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=1
+
+[mysqld.1]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+innodb_fast_shutdown=0
+innodb_undo_tablespaces=0
+
+[mysqld.2]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+innodb_fast_shutdown=0
+innodb_undo_tablespaces=3
+loose_innodb_log_file_buffering
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream
+
+[mysqld.1]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=11
+
+[mysqld.2]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=12

mysql-test/suite/galera/t/galera_binlog_in_engine_sst_mariabackup.test

@@ -0,0 +1,60 @@
+#
+# State transfers (IST and mariabackup SST) with the binlog implemented in InnoDB
+# (--binlog-storage-engine=innodb): the joiner is shut down, has its data
+# directory removed, or is killed, while the cluster keeps committing
+# transactions. After each state transfer, the joiner must have the same
+# data and binlog the same GTIDs as the donor.
+#
+--source include/have_innodb_binlog.inc
+--source include/galera_cluster.inc
+--connection node_1
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+--connection node_2
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--connection node_1
+select @@innodb_undo_tablespaces;
+
+--connection node_2
+select @@innodb_undo_tablespaces;
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+--connection node_2
+--echo Shutting down server ...
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+--connection node_2
+--echo Starting server ...
+--source include/start_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+--connection node_2
+select @@innodb_undo_tablespaces;
+
+call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=3 because previous shutdown was not with innodb_fast_shutdown=0");
+
+# Restore original auto_increment_offset values.
+--source suite/galera/include/galera_binlog_in_engine_check_gtid_history.inc
+
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc

mysql-test/suite/galera/t/galera_binlog_in_engine_sst_rsync.cnf

@@ -0,0 +1,26 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=rsync
+
+[mysqld.1]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+
+[mysqld.2]
+wsrep_provider_options='pc.ignore_sb=true;repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+
+[mysqld.1]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=11
+
+[mysqld.2]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=12

mysql-test/suite/galera/t/galera_binlog_in_engine_sst_rsync.test

@@ -0,0 +1,27 @@
+#
+# State transfers (IST and rsync SST) with the binlog implemented in InnoDB
+# (--binlog-storage-engine=innodb): the joiner is shut down, has its data
+# directory removed, or is killed, while the cluster keeps committing
+# transactions. After each state transfer, the joiner must have the same
+# data and binlog the same GTIDs as the donor.
+#
+--source include/have_innodb_binlog.inc
+--source include/galera_cluster.inc
+--connection node_1
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+--connection node_2
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+--source suite/galera/include/galera_binlog_in_engine_check_gtid_history.inc
+
+--source include/auto_increment_offset_restore.inc

scripts/wsrep_sst_mariabackup.sh

@@ -1563,10 +1563,18 @@ else # joiner
             # from which IST resumes, which keeps the joiner's binary log in
             # lockstep with the rest of the cluster.
             #
+            # The binlog files of --binlog-storage-engine=innodb
+            # (binlog-NNNNNN.ibb) are not removed: mariabackup copies them
+            # consistently with the InnoDB data, and the InnoDB redo log
+            # refers to them.
+            #
             binlogs=()
             if [ -f 'mariadb_backup_binlog_info' ]; then
                 while read bin_string || [ -n "$bin_string" ]; do
                     bin_file=$(echo "$bin_string" | cut -f1)
+                    case "$bin_file" in
+                    binlog-[0-9]*.ibb) continue ;;
+                    esac
                     if [ -n "$bin_file" -a -f "$bin_file" ]; then
                         binlogs+=("$bin_file")
                     fi

sql/log_cache.cc

@@ -60,13 +60,15 @@ bool init_binlog_cache_dir()
       2 /* prefix */ + 10 /* max len of thread_id */ + 1 /* underline */;
 
   /*
-    Even if the binary log is disabled (and thereby we wouldn't use the binlog
-    cache), we need to try to build the directory name, so if it exists while
-    the binlog is off (e.g. due to a previous run of mariadbd, or an SST), we
-    can delete it.
+    Even if the binary log is disabled, or implemented in a storage engine
+    (--binlog-storage-engine), and thereby we wouldn't use the binlog cache
+    dir, we need to try to build the directory name, so if it exists (e.g.
+    due to a previous run of mariadbd, or an SST from a server with a file
+    binlog), we can delete it.
   */
+  const bool use_cache_dir= opt_bin_log && !opt_binlog_engine_hton;
   dirname_part(binlog_cache_dir,
-               opt_bin_log ? log_bin_basename : opt_log_basename, &length);
+               use_cache_dir ? log_bin_basename : opt_log_basename, &length);
   /*
     Must ensure the full name of the tmp file is shorter than FN_REFLEN, to
     avoid overflowing the name buffer in write and commit.
@@ -85,17 +87,18 @@ bool init_binlog_cache_dir()
   MY_DIR *dir_info= my_dir(binlog_cache_dir, MYF(0));
 
   /*
-    If the binlog cache dir exists, yet binlogging is disabled, delete the
+    If the binlog cache dir exists, yet it is not used, delete the
     directory and skip the initialization logic.
   */
-  if (!opt_bin_log)
+  if (!use_cache_dir)
   {
     if (dir_info)
     {
       sql_print_information(
-          "Found binlog cache dir '%s', yet binary logging is "
-          "disabled. Deleting directory.",
-          binlog_cache_dir);
+          "Found binlog cache dir '%s', yet binary logging is %s. "
+          "Deleting directory.",
+          binlog_cache_dir,
+          opt_bin_log ? "implemented in a storage engine" : "disabled");
       my_dirend(dir_info);
       my_rmtree(binlog_cache_dir, MYF(0));
     }

sql/mysqld.cc

@@ -5926,7 +5926,7 @@ static int init_server_components()
       unireg_abort(1);
   }
 
-  if (!binlog_engine_used && unlikely(init_binlog_cache_dir()))
+  if (unlikely(init_binlog_cache_dir()))
     unireg_abort(1);
 
 #ifdef HAVE_REPLICATION