MDEV-34705: Binlog-in-engine: mariadb-backup integration

InnoDB binlog files are now backed up along with other InnoDB data by
mariadb-backup.

The files are copied after backup locks have been released. Backup files
created later than the backup LSN are skipped. Then during --prepare, any
data missing from the hot-copied binlog files will be restored by the
binlog recovery code, and any excess data written after the backup LSN will
be zeroed out.

A couple test cases test taking a consistent backup of a server with active
traffic during the backup, by provisioning a slave from the restored binlog
position and checking that the slave can replicate from the original master
and get identical data.

Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>

mysql-test/suite/binlog_in_engine/include/mariabackup_slave_provision.inc

@@ -0,0 +1,105 @@
+--source include/have_innodb_binlog.inc
+# Test does a lot of queries that take a lot of CPU under Valgrind.
+--source include/not_valgrind.inc
+
+RESET MASTER;
+
+let $basedir=$MYSQLTEST_VARDIR/tmp/backup;
+
+CREATE TABLE t1(a varchar(60) PRIMARY KEY, b VARCHAR(60)) ENGINE INNODB;
+INSERT INTO t1 VALUES(1, NULL);
+CREATE TABLE t2 (val INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0);
+
+--disable_query_log
+--delimiter //
+CREATE PROCEDURE gen_load()
+  MODIFIES SQL DATA
+  BEGIN
+    DECLARE i INT;
+    DECLARE flag TYPE OF t2.val;
+    SET i = 0;
+    load_loop: LOOP
+      SELECT val INTO flag FROM t2;
+      IF NOT (flag=0) THEN
+        LEAVE load_loop;
+      END IF;
+      START TRANSACTION;
+      INSERT INTO t1 VALUES (CONCAT("AbAdCaFe", LPAD(i, 6, "0")), @@SESSION.last_gtid);
+      COMMIT;
+      SET i = i + 1;
+    END LOOP;
+  END
+//
+--delimiter ;
+--enable_query_log
+
+connect (con1,localhost,root,,);
+--echo *** Start a background load...
+send CALL gen_load();
+
+--connection default
+--echo *** Doing backup...
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir $backup_args
+--echo *** Doing prepare...
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --prepare --target-dir=$basedir
+
+--echo *** Stop the background load...
+UPDATE t2 SET val=1;
+--connection con1
+reap;
+--connection default
+disconnect con1;
+--let $count_master= `SELECT COUNT(*) FROM t1`
+
+--echo *** Provision a new slave from the backup
+--connect (server2,127.0.0.1,root,,,$SERVER_MYPORT_2)
+--let $datadir_2= `SELECT @@datadir`
+
+--echo *** Stopping provisioned server
+--source include/shutdown_mysqld.inc
+
+--echo *** Removing old datadir for provisioned server
+--rmdir $datadir_2
+
+--echo *** Provision new server from backup
+--exec $XTRABACKUP --copy-back --datadir=$datadir_2 --target-dir=$basedir $copy_back_args
+
+--let $restart_parameters= --skip-slave-start
+--source include/start_mysqld.inc
+
+--let $gtid_pos= `SELECT @@GLOBAL.gtid_binlog_pos`
+
+--replace_result $gtid_pos GTID_POS
+eval SET GLOBAL gtid_slave_pos= '$gtid_pos';
+--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1
+eval CHANGE MASTER TO
+  master_ssl_verify_server_cert=0,
+  master_port=$SERVER_MYPORT_1, master_host='127.0.0.1', master_user='root',
+  master_use_gtid= Slave_pos;
+START SLAVE;
+
+--connection default
+--source include/save_master_gtid.inc
+
+--connection server2
+--source include/sync_with_master_gtid.inc
+--let $count_slave= `SELECT COUNT(*) FROM t1`
+if ($count_master != $count_slave) {
+  --echo *** ERROR: Table on master has $count_master rows, but table on provisioned slave has $count_slave rows
+  --die Row difference on provisioned slave.
+}
+
+# Cleanup
+
+--connection server2
+STOP SLAVE;
+RESET SLAVE ALL;
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;
+
+--connection default
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;
+
+rmdir $basedir;

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_binlog_dir.cnf

@@ -0,0 +1,5 @@
+!include my.cnf
+
+[mysqld]
+binlog_directory=binlogs
+loose_innodb_log_file_size=96M

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_binlog_dir.result

@@ -0,0 +1,40 @@
+RESET MASTER;
+CREATE TABLE t1(a varchar(60) PRIMARY KEY, b VARCHAR(60)) ENGINE INNODB;
+INSERT INTO t1 VALUES(1, NULL);
+CREATE TABLE t2 (val INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0);
+connect  con1,localhost,root,,;
+*** Start a background load...
+CALL gen_load();
+connection default;
+*** Doing backup...
+*** Doing prepare...
+*** Stop the background load...
+UPDATE t2 SET val=1;
+connection con1;
+connection default;
+disconnect con1;
+*** Provision a new slave from the backup
+connect  server2,127.0.0.1,root,,,$SERVER_MYPORT_2;
+*** Stopping provisioned server
+*** Removing old datadir for provisioned server
+*** Provision new server from backup
+# restart: --skip-slave-start
+SET GLOBAL gtid_slave_pos= 'GTID_POS';
+CHANGE MASTER TO
+master_ssl_verify_server_cert=0,
+master_port=SERVER_MYPORT_1, master_host='127.0.0.1', master_user='root',
+master_use_gtid= Slave_pos;
+START SLAVE;
+connection default;
+include/save_master_gtid.inc
+connection server2;
+include/sync_with_master_gtid.inc
+connection server2;
+STOP SLAVE;
+RESET SLAVE ALL;
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;
+connection default;
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_binlog_dir.test

@@ -0,0 +1,4 @@
+--source include/not_embedded.inc
+--let $backup_args=
+--let $copy_back_args= --binlog-directory=binlogs
+--source include/mariabackup_slave_provision.inc

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_nolock.cnf

@@ -0,0 +1,4 @@
+!include my.cnf
+
+[mysqld]
+loose_innodb_log_file_size=96M

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_nolock.result

@@ -0,0 +1,40 @@
+RESET MASTER;
+CREATE TABLE t1(a varchar(60) PRIMARY KEY, b VARCHAR(60)) ENGINE INNODB;
+INSERT INTO t1 VALUES(1, NULL);
+CREATE TABLE t2 (val INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0);
+connect  con1,localhost,root,,;
+*** Start a background load...
+CALL gen_load();
+connection default;
+*** Doing backup...
+*** Doing prepare...
+*** Stop the background load...
+UPDATE t2 SET val=1;
+connection con1;
+connection default;
+disconnect con1;
+*** Provision a new slave from the backup
+connect  server2,127.0.0.1,root,,,$SERVER_MYPORT_2;
+*** Stopping provisioned server
+*** Removing old datadir for provisioned server
+*** Provision new server from backup
+# restart: --skip-slave-start
+SET GLOBAL gtid_slave_pos= 'GTID_POS';
+CHANGE MASTER TO
+master_ssl_verify_server_cert=0,
+master_port=SERVER_MYPORT_1, master_host='127.0.0.1', master_user='root',
+master_use_gtid= Slave_pos;
+START SLAVE;
+connection default;
+include/save_master_gtid.inc
+connection server2;
+include/sync_with_master_gtid.inc
+connection server2;
+STOP SLAVE;
+RESET SLAVE ALL;
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;
+connection default;
+DROP PROCEDURE gen_load;
+DROP TABLE t1, t2;

mysql-test/suite/binlog_in_engine/mariabackup_slave_provision_nolock.test

@@ -0,0 +1,4 @@
+--source include/not_embedded.inc
+--let $backup_args= --no-lock
+--let $copy_back_args=
+--source include/mariabackup_slave_provision.inc