Galera4

mysql-test/suite/wsrep/disabled.def

@@ -1,2 +1,3 @@
 wsrep.foreign_key : Sporadic failure "WSREP has not yet prepared node for application use"
 wsrep.pool_of_threads : Sporadic failure "WSREP has not yet prepared node for application use"
+wsrep.variables : Global wsrep_on manipulation causes debug asserts

mysql-test/suite/wsrep/my.cnf

@@ -1,10 +1,8 @@
 # Use default setting for mysqld processes
 !include include/default_mysqld.cnf
 
-[mysqld]
-wsrep-on=1
-
 [mysqld.1]
+wsrep-on=OFF
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port

mysql-test/suite/wsrep/r/wsrep-recover,binlogon.rdiff

@@ -0,0 +1,28 @@
+--- r/wsrep-recover.result	2019-01-11 16:22:46.329012579 +0200
++++ r/wsrep-recover.reject	2019-01-11 16:23:55.313137675 +0200
+@@ -48,19 +48,17 @@
+ SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached_1";
+ connection default;
+ # Kill the server
+-Expect seqno 9
+-9
++Expect seqno 7
++7
+ disconnect con1;
+ disconnect con2;
+ disconnect con_ctrl;
+ connection default;
+-SELECT VARIABLE_VALUE `expect 10` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+-expect 10
+-10
+-Expect rows 5, 9, 10
++SELECT VARIABLE_VALUE `expect 8` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
++expect 8
++8
++Expect row 5
+ SELECT * FROM t1;
+ f1
+ 5
+-9
+-10
+ DROP TABLE t1;

mysql-test/suite/wsrep/r/wsrep-recover.result

@@ -0,0 +1,66 @@
+# Kill the server
+Expect seqno 1
+1
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+# Kill the server
+Expect seqno 3
+3
+INSERT INTO t1 VALUES (5);
+# Kill the server
+Expect seqno 5
+5
+SELECT VARIABLE_VALUE `expect 6` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+expect 6
+6
+connect con1, localhost, root;
+SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue";
+INSERT INTO t1 VALUES (7);
+connect con_ctrl, localhost, root;
+SET DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
+connect con2, localhost, root;
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached WAIT_FOR continue";
+INSERT INTO t1 VALUES (8);
+connection con_ctrl;
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached";
+connection default;
+# Kill the server
+Expect seqno 6
+6
+disconnect con1;
+disconnect con2;
+disconnect con_ctrl;
+connection default;
+SELECT VARIABLE_VALUE `expect 7` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+expect 7
+7
+connect con1, localhost, root;
+SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached_1 WAIT_FOR continue_before_commit_order_1";
+INSERT INTO t1 VALUES (9);
+connect con_ctrl, localhost, root;
+SET DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
+connect con2, localhost, root;
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached_2 WAIT_FOR continue_before_commit_order_2";
+INSERT INTO t1 VALUES (10);
+connection con_ctrl;
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached_2";
+SET DEBUG_SYNC = "now SIGNAL continue_after_certification";
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached_1";
+connection default;
+# Kill the server
+Expect seqno 9
+9
+disconnect con1;
+disconnect con2;
+disconnect con_ctrl;
+connection default;
+SELECT VARIABLE_VALUE `expect 10` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+expect 10
+10
+Expect rows 5, 9, 10
+SELECT * FROM t1;
+f1
+5
+9
+10
+DROP TABLE t1;

mysql-test/suite/wsrep/suite.pm

@@ -9,9 +9,9 @@ return "Not run for embedded server" if $::opt_embedded_server;
 return "WSREP is not compiled in" unless defined $::mysqld_variables{'wsrep-on'};
 
 my ($provider) = grep { -f $_ } $ENV{WSREP_PROVIDER},
-                                "/usr/lib64/galera-3/libgalera_smm.so",
+                                "/usr/lib64/galera-4/libgalera_smm.so",
                                 "/usr/lib64/galera/libgalera_smm.so",
-                                "/usr/lib/galera-3/libgalera_smm.so",
+                                "/usr/lib/galera-4/libgalera_smm.so",
                                 "/usr/lib/galera/libgalera_smm.so";
 
 return "No wsrep provider library" unless -f $provider;

mysql-test/suite/wsrep/t/binlog_format.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address=gcomm://
+innodb_autoinc_lock_mode=2
+

mysql-test/suite/wsrep/t/binlog_format.opt

@@ -1 +0,0 @@
---innodb_autoinc_lock_mode=2 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm://

mysql-test/suite/wsrep/t/binlog_format.test

@@ -1,3 +1,4 @@
+--source include/have_innodb.inc
 --source include/have_wsrep_provider.inc
 --source include/have_binlog_format_row.inc
 

mysql-test/suite/wsrep/t/mdev_10186.cnf

@@ -0,0 +1,6 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=OFF
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address=gcomm://

mysql-test/suite/wsrep/t/mdev_10186.opt

@@ -1 +0,0 @@
---wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=0

mysql-test/suite/wsrep/t/mdev_10186.test

@@ -1,3 +1,4 @@
+--source include/have_innodb.inc
 --source include/have_wsrep_provider.inc
 --source include/have_binlog_format_row.inc
 

mysql-test/suite/wsrep/t/mdev_6832.cnf

@@ -0,0 +1,7 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address=gcomm://
+

mysql-test/suite/wsrep/t/mdev_6832.opt

@@ -1 +0,0 @@
---wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=1

mysql-test/suite/wsrep/t/mdev_6832.test

@@ -1,3 +1,4 @@
+--source include/have_innodb.inc
 --source include/have_wsrep_provider.inc
 --source include/have_binlog_format_row.inc
 

mysql-test/suite/wsrep/t/mdev_7798.cnf

@@ -0,0 +1,7 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address=gcomm://
+

mysql-test/suite/wsrep/t/mdev_7798.opt

@@ -1 +0,0 @@
---wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=1 

mysql-test/suite/wsrep/t/mdev_7798.test

@@ -1,3 +1,4 @@
+--source include/have_innodb.inc
 --source include/have_wsrep_provider.inc
 --source include/have_binlog_format_row.inc
 

mysql-test/suite/wsrep/t/pool_of_threads.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address=gcomm://
+thread_handling=pool-of-threads
+

mysql-test/suite/wsrep/t/pool_of_threads.opt

@@ -1 +0,0 @@
---innodb_autoinc_lock_mode=2 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --thread_handling=pool-of-threads

mysql-test/suite/wsrep/t/wsrep-recover-step.inc

@@ -0,0 +1,41 @@
+#
+# Macro to run wsrep recovery step. This is adapted from
+# suite/galera/include/galera_wsrep_recover.inc, with additional
+# option to pass binlog argument to recovery command. The macro
+# returns recovered position split in uuid and seqno parts.
+#
+# Arguments:
+#
+# wsrep_recover_binlog_opt - Binlog options to recovery command
+#
+# Return:
+#
+# wsrep_recover_start_position_uuid - UUID corresponding to recovered position
+# wsrep_recover_start_position_seqno - seqno corresponding to recovered position
+#
+
+--exec $MYSQLD --defaults-group-suffix=.1 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_binlog_opt --core-file > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+
+--perl
+	use strict;
+	my $wsrep_start_position = `grep 'WSREP: Recovered position:' $ENV{MYSQL_TMP_DIR}/galera_wsrep_recover.log | sed 's/.*WSREP\:\ Recovered\ position://' | sed 's/^[ \t]*//'`;
+	chomp($wsrep_start_position);
+        die if $wsrep_start_position eq '';
+	open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/galera_wsrep_start_position.inc") or die;
+	my ($uuid, $seqno) = split /:/, $wsrep_start_position;
+	print FILE "--let \$wsrep_recover_start_position_uuid = $uuid\n";
+	print FILE "--let \$wsrep_recover_start_position_seqno = $seqno\n";
+	close FILE;
+EOF
+
+--source $MYSQL_TMP_DIR/galera_wsrep_start_position.inc
+
+if ($wsrep_recover_start_position_uuid == '') {
+   --die "Could not obtain start_position_uuid."
+}
+
+if ($wsrep_recover_start_position_seqno == '') {
+   --die "Could not obtain start_position_seqno."
+}
+
+--remove_file $MYSQL_TMP_DIR/galera_wsrep_start_position.inc

mysql-test/suite/wsrep/t/wsrep-recover.cnf

@@ -0,0 +1,9 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+binlog-format=ROW
+innodb-flush-log-at-trx-commit=1
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+innodb-autoinc-lock-mode=2

mysql-test/suite/wsrep/t/wsrep-recover.combinations

@@ -0,0 +1,4 @@
+[binlogon]
+log-bin
+
+[binlogoff]

mysql-test/suite/wsrep/t/wsrep-recover.test

@@ -0,0 +1,204 @@
+#
+# Verify that the wsrep XID gets updated in InnoDB rollback segment
+# properly and can be recovered with --wsrep-recover
+#
+# The test runs the following scenarios:
+#
+# 1) The server is started but no SQL is run
+# 2) DDL is executed
+# 3) INSERT is executed
+# 4) Two INSERTs are executed so that the first one in order will be
+#    blocked after certification and the second one before entering
+#    commit order critical section.
+# 5) Two DMLs are executed so that the prepare step is run out of order.
+#    Both transactions are blocked before commit order critical section.
+#
+# After each scenario server is killed and the recovered position
+# is validated.
+#
+
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+--source include/have_wsrep_provider.inc
+--source include/have_debug_sync.inc
+
+#
+# Binlog option for recovery run. This must be set in the test because
+# combinations file causes log-bin option to be set from command line,
+# not via my.cnf.
+#
+--let $log_bin = `SELECT @@log_bin`
+if ($log_bin) {
+--let $wsrep_recover_binlog_opt = --log-bin
+}
+
+#
+# Scenario 1
+# The expected recovered seqno is 1 corresponding to initial cluster
+# configuration change.
+#
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 1
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Senario 2
+# The expected recovered seqno is 3 corresponding to two configuration
+# change events and CREATE TABLE.
+#
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 3
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Scenario 3
+# The expected recovered seqno is 5 corresponding to three configuration
+# change events, CREATE TABLE and INSERT.
+#
+# The expected wsrep_last_committed after the server is restarted is 6.
+#
+
+INSERT INTO t1 VALUES (5);
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 5
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+SELECT VARIABLE_VALUE `expect 6` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+
+#
+# Scenario 4
+#
+# This will cause the following
+#
+# Seqno 7 - the first INSERT is blocked after it is certified but before
+#           it gets prepared
+# Seqno 8 - the second INSERT is blocked before it will be ordered for
+#           commit, so it becomes prepared
+#
+# As an outcome, the recovery process should return seqno 6 because
+# the range of prepared transactions found after the crash recovery
+# is not continuous up to 8.
+#
+# The expected wsrep_last_committed after server is restarted is 7.
+#
+
+# Send INSERT which will block after certification
+--connect con1, localhost, root
+SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue";
+--send INSERT INTO t1 VALUES (7)
+
+--connect con_ctrl, localhost, root
+SET DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
+
+# Send INSERT which will block before commit order critical section
+--connect con2, localhost, root
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached WAIT_FOR continue";
+--send INSERT INTO t1 VALUES (8)
+
+--connection con_ctrl
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached";
+
+--connection default
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 6
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+--disconnect con1
+--disconnect con2
+--disconnect con_ctrl
+--connection default
+
+SELECT VARIABLE_VALUE `expect 7` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+
+#
+# Scenario 5
+#
+# This scenario will run two INSERTs in parallel so that they are
+# prepared out of order. The execution is stopped before commit
+# and the server is killed. The outcome of this scenario depends
+# on binlog settings:
+#
+# If binlog is off, the transactions will be recovered from InnoDB and
+# committed during recovery. The expected recovered seqno is 9, the
+# expected wsrep_last_committed after server is restarted is 10.
+#
+# If binlog is on, the transactions will be recovered from InnoDB but
+# will be rolled back since they are not logged yet in binlog. The
+# expected recovered seqno is 7, the expected wsrep_last_committed
+# after server is restarted is 8.
+#
+
+--connect con1, localhost, root
+SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached_1 WAIT_FOR continue_before_commit_order_1";
+--send INSERT INTO t1 VALUES (9)
+
+--connect con_ctrl, localhost, root
+SET DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
+
+--connect con2, localhost, root
+SET DEBUG_SYNC = "wsrep_before_commit_order_enter SIGNAL before_commit_order_reached_2 WAIT_FOR continue_before_commit_order_2";
+--send INSERT INTO t1 VALUES (10)
+
+--connection con_ctrl
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached_2";
+SET DEBUG_SYNC = "now SIGNAL continue_after_certification";
+SET DEBUG_SYNC = "now WAIT_FOR before_commit_order_reached_1";
+
+--connection default
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+if ($log_bin) {
+   --echo Expect seqno 7
+}
+if (!$log_bin) {
+   --echo Expect seqno 9
+}
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+--disconnect con1
+--disconnect con2
+--disconnect con_ctrl
+--connection default
+
+if ($log_bin) {
+   SELECT VARIABLE_VALUE `expect 8` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+}
+if (!$log_bin) {
+   SELECT VARIABLE_VALUE `expect 10` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+}
+
+#
+# Final sanity check: The successful inserts into t1 should result
+if ($log_bin) {
+   --echo Expect row 5
+}
+if (!$log_bin) {
+   --echo Expect rows 5, 9, 10
+}
+SELECT * FROM t1;
+
+DROP TABLE t1;