MDEV-33799: mysql_manager_submit Segfault at Startup Still Possible During Recovery

MDEV-26473 fixed a segmentation fault at startup between the handle
manager thread and the binlog background thread, such that the
binlog background thread could be started and submit a job to the
handle manager, before it had initialized. Where MDEV-26473 made it
so the handle manager would initialize before the main thread
started the normal binary logs, it did not account for the recovery
case. That is, there is still a possibility of a segmentation fault
when a server is recovering using the binary logs such that it can
open the binary logs, start the binlog background thread, and submit
a job to the handle manager before it is initialized.

This patch fixes this by moving the initialization of the mysql
handler manager to happen prior to recovery.

Reviewed By:
============
Andrei Elkin <andrei.elkin@mariadb.com>

mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result

@@ -17,6 +17,36 @@ include/sync_with_master_gtid.inc
 include/rpl_restart_server.inc [server_number=2 parameters: --debug_dbug="+d,delay_start_handle_manager"]
 include/start_slave.inc
 #
+# MDEV-33799
+# Ensure that when the binary log is used for recovery (as tc log), that
+# the recovery process cannot start the binlog background thread before
+# the mysql handle manager has started.
+connection slave;
+# Add test suppresssions so crash recovery messages don't fail the test
+set session sql_log_bin=0;
+call mtr.add_suppression("mariadbd: Got error '145.*");
+call mtr.add_suppression("Checking table:.*");
+call mtr.add_suppression("mysql.gtid_slave_pos:.*hasn't closed the table properly");
+call mtr.add_suppression("Can't init tc log");
+call mtr.add_suppression("Aborting");
+set session sql_log_bin=1;
+# Create slave-side only table
+create table t2 (a int) engine=innodb;
+# Crash mariadbd when binlogging transaction to corrupt database state
+connection slave1;
+set @@session.debug_dbug="+d,crash_before_writing_xid";
+insert into t2 values (1);
+connection slave;
+connection slave1;
+Got one of the listed errors
+# Restart mariadbd in recovery mode. Note --tc-heuristic-recover
+# forces mysqld to exit with error, so we run mariadbd via CLI
+# MYSQLD_LAST_CMD --debug_dbug="+d,delay_start_handle_manager" --tc-heuristic-recover=COMMIT
+connection server_2;
+connection slave1;
+connection slave;
+include/start_slave.inc
+#
 # Cleanup
 #
 connection master;
@@ -24,4 +54,5 @@ drop table t1;
 include/save_master_gtid.inc
 connection slave;
 include/sync_with_master_gtid.inc
+drop table t2;
 include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test

@@ -18,8 +18,14 @@
 # associated with this test should enforce that the binlog background thread is
 # not created before the handle manager is initialized.
 #
+#   Addendum 1) This test is extended for MDEV-33799, as the original fix
+# left out the possibility that the binlog background thread can be
+# started during recovery if the binary log is used as the transaction
+# coordinator. This resulted in similar segfaults as seen by MDEV-26473.
+#
 # References:
 #  MDEV-26473 mysqld got exception 0xc0000005 (rpl_slave_state/rpl_load_gtid_slave_state)
+#  MDEV-33799 mysql_manager_submit Segfault at Startup Still Possible During Recovery
 #
 
 --source include/have_debug.inc
@@ -53,6 +59,63 @@ create table t1 (a int);
 --source include/start_slave.inc
 
 
+--echo #
+--echo # MDEV-33799
+--echo # Ensure that when the binary log is used for recovery (as tc log), that
+--echo # the recovery process cannot start the binlog background thread before
+--echo # the mysql handle manager has started.
+--connection slave
+
+--echo # Add test suppresssions so crash recovery messages don't fail the test
+set session sql_log_bin=0;
+call mtr.add_suppression("mariadbd: Got error '145.*");
+call mtr.add_suppression("Checking table:.*");
+call mtr.add_suppression("mysql.gtid_slave_pos:.*hasn't closed the table properly");
+call mtr.add_suppression("Can't init tc log");
+call mtr.add_suppression("Aborting");
+set session sql_log_bin=1;
+
+--echo # Create slave-side only table
+create table t2 (a int) engine=innodb;
+
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+wait
+EOF
+
+--echo # Crash mariadbd when binlogging transaction to corrupt database state
+--connection slave1
+set @@session.debug_dbug="+d,crash_before_writing_xid";
+--send insert into t2 values (1)
+
+--connection slave
+--source include/wait_until_disconnected.inc
+
+--connection slave1
+--error 2013,ER_CONNECTION_KILLED
+--reap
+
+--echo # Restart mariadbd in recovery mode. Note --tc-heuristic-recover
+--echo # forces mysqld to exit with error, so we run mariadbd via CLI
+--echo # MYSQLD_LAST_CMD --debug_dbug="+d,delay_start_handle_manager" --tc-heuristic-recover=COMMIT
+--error 1
+--exec $MYSQLD_LAST_CMD --debug_dbug="+d,delay_start_handle_manager" --tc-heuristic-recover=COMMIT
+
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+restart
+EOF
+
+--connection server_2
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--connection slave1
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--connection slave
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--source include/start_slave.inc
+
+
 --echo #
 --echo # Cleanup
 --echo #
@@ -63,5 +126,6 @@ drop table t1;
 
 --connection slave
 --source include/sync_with_master_gtid.inc
+drop table t2;
 
 --source include/rpl_end.inc