MDEV-25616 XA PREPARE event group is not binlogged when..

the only query of the XA transaction is on a non-transactional table
errors out:

XA BEGIN 'x';
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES (1),(1);
XA END 'x';
XA PREPARE 'x';

The binlogging pattern is correctly started as expected with
the errored-out Query or its ROW format events, but there is
no empty XA_prepare_log_event group.
The following

XA COMMIT 'x';

therefore should not be logged either, but it does.

The bug is fixed with proper maintaining of a read-write binlog hton
property and use it to enforce correct binlogging decisions.
Specifically in the bug description case XA COMMIT won't be binlogged
in both when given in the same connection and externally after disconnect.
The same continue to apply to an empty XA that do not change any data in all
transactional engines involved.

mysql-test/suite/rpl/include/rpl_xa_empty_transaction.inc

@@ -0,0 +1,10 @@
+#
+#   Helper file to run each empty-due-to-err XA transaction test case both with
+# and without detaching from the connection when the transaction is prepared.
+#
+
+--let $use_disconnect=0
+--source rpl_xa_empty_transaction_test_case.inc
+
+--let $use_disconnect=1
+--source rpl_xa_empty_transaction_test_case.inc

mysql-test/suite/rpl/include/rpl_xa_empty_transaction_test_case.inc

@@ -0,0 +1,131 @@
+#
+#   Helper script to create an XA transaction and validate it was not
+# binlogged
+#
+# Parameters
+#   $xa_completion_action     : The action to end the XA transaction, either
+#                               COMMIT or ROLLBACK
+#   $trx_statements           : A comma separated list specifying how to build
+#                               the statements of the transaction. Each item in
+#                               the list is either T (for transactional) or N
+#                               (for non-transactional). An empty list will not
+#                               add any statements to the transaction.
+#   $use_disconnect           : When TRUE, disconnect after preparing the XA
+#                               transaction to test the detach/rollback case
+#
+
+#
+# Setup
+--let $generic_assert_text= should not binlog XA transaction
+
+--connection server_1
+--let server_1_datadir=`select @@datadir`
+
+--connection server_2
+--let server_2_datadir=`select @@datadir`
+
+--connection server_3
+--let server_3_datadir=`select @@datadir`
+
+--let assert_file=$MYSQLTEST_VARDIR/tmp/binlog_decoded.out
+
+--connection server_1
+CREATE TABLE tm (a INT PRIMARY KEY) engine=myisam;
+CREATE TABLE ti (a INT PRIMARY KEY) engine=innodb;
+--source include/save_master_gtid.inc
+
+--connection server_3
+--source include/sync_with_master_gtid.inc
+
+--connection server_1
+
+if ($use_disconnect)
+{
+    --source include/count_sessions.inc
+    --connect(con1,localhost,root,,)
+}
+
+XA START 'x';
+--let $_stmt_items= $trx_statements
+--let $_ctr= 1
+while($_stmt_items)
+{
+  --let $_cur_stmt= `SELECT SUBSTRING_INDEX('$_stmt_items', ',', 1)`
+  --let $_stmt_items= `SELECT LTRIM(SUBSTRING('$_stmt_items', LENGTH('$_cur_stmt') + 2))`
+
+    if (`SELECT strcmp("$_cur_stmt","T") = 0`)
+    {
+        --let $target_table= ti
+    }
+
+    if (`SELECT strcmp("$_cur_stmt","N") = 0`)
+    {
+        --let $target_table= tm
+    }
+
+    --error ER_DUP_ENTRY
+    --eval INSERT INTO $target_table VALUES ($_ctr),($_ctr);
+    inc $_ctr;
+
+}
+XA END 'x';
+XA PREPARE 'x';
+
+if ($use_disconnect)
+{
+    --disconnect con1
+    --connection server_1
+    --source include/wait_until_count_sessions.inc
+    XA RECOVER;
+
+    --error ER_XA_RBROLLBACK
+    --eval XA $xa_completion_action 'x';
+}
+if (!$use_disconnect)
+{
+    --eval XA $xa_completion_action 'x';
+}
+
+--source include/save_master_gtid.inc
+
+--let binlog_filename= query_get_value(SHOW MASTER STATUS, File, 1)
+FLUSH LOGS;
+
+--echo # MYSQL_BINLOG server_1_datadir/binlog_filename --result-file=assert_file
+--exec $MYSQL_BINLOG $server_1_datadir/$binlog_filename --result-file=$assert_file
+
+--let assert_text= server_1 $generic_assert_text
+--let assert_count= 0
+--let assert_select= XA START|XA END|XA PREPARE|XA COMMIT|XA ROLLBACK
+--source include/assert_grep.inc
+
+--connection server_2
+--source include/sync_with_master_gtid.inc
+--let binlog_filename= query_get_value(SHOW MASTER STATUS, File, 1)
+FLUSH LOGS;
+
+--echo # MYSQL_BINLOG server_2_datadir/binlog_filename --result-file=assert_file
+--exec $MYSQL_BINLOG $server_2_datadir/$binlog_filename --result-file=$assert_file
+
+--let assert_text= server_2 $generic_assert_text
+--source include/assert_grep.inc
+
+--connection server_3
+--source include/sync_with_master_gtid.inc
+--let binlog_filename= query_get_value(SHOW MASTER STATUS, File, 1)
+FLUSH LOGS;
+
+--echo # MYSQL_BINLOG server_3_datadir/binlog_filename --result-file=assert_file
+--exec $MYSQL_BINLOG $server_3_datadir/$binlog_filename --result-file=$assert_file
+
+--let assert_text= server_3 $generic_assert_text
+--source include/assert_grep.inc
+
+#
+# Cleanup
+--connection server_1
+DROP TABLE ti,tm;
+--source include/save_master_gtid.inc
+
+--connection server_3
+--source include/sync_with_master_gtid.inc

mysql-test/suite/rpl/t/rpl_xa_empty_transaction.cnf

@@ -0,0 +1,18 @@
+!include include/default_mysqld.cnf
+
+[mysqld.1]
+log-slave-updates
+innodb
+
+[mysqld.2]
+log-slave-updates
+innodb
+
+[mysqld.3]
+log-slave-updates
+innodb
+
+[ENV]
+SERVER_MYPORT_1= @mysqld.1.port
+SERVER_MYPORT_2= @mysqld.2.port
+SERVER_MYPORT_3= @mysqld.3.port

mysql-test/suite/rpl/t/rpl_xa_empty_transaction.test

@@ -0,0 +1,175 @@
+#
+# Purpose:
+#   This test ensures consistency in binlogging behavior for XA transactions
+# that have all statements error and rollback, effectively leaving an "empty"
+# transaction. In such cases, an empty XA transaction should be binlogged. The
+# bug reported by MDEV-25616 revealed that an "empty" XA transaction would
+# binlog an XA ROLLBACK or XA COMMIT event without a preceding setup, i.e.
+# XA START through XA PREPARE. The bug presented differently for XA
+# transactions consisting of transactional and non-transactional statements.
+# Therefore, this test validates that an entire XA transaction is binlogged
+# for different combinations of transactional or non-transactional statements.
+# Note that the behavior changes when binlogging empty XA transactions
+# depending on the binlog_row_format variables. That is, when the content of
+# the transaction consists of errored transactional statements, in row format,
+# an empty XA transaction will be binlogged; however, in mixed and statement
+# formats, nothing will be written into the binary log.
+#
+# Methodology:
+#   Create XA transactions with various combinations of erroring transactional
+# or non-transactional statements. The binary log is examined to ensure all
+# XA components are written. Chain replication is used, i.e.
+# (primary->replica->replica), to ensure replica binlogging is consistent with
+# manual execution. The transactional and non-transactional tables use InnoDB
+# and MyISAM, respectively.
+#
+# Parameters
+#   $expect_transactional_xa_binlog : Boolean indicating whether or not an
+#       errored transactional statement should result in XA statements written
+#       into the binary log.
+#
+# References:
+#   MDEV-25616: Binlog event for XA COMMIT is generated without matching
+#               XA START, replication aborts
+#
+--source include/have_log_bin.inc
+
+--let $rpl_server_count= 3
+--let $rpl_topology= 1->2->3
+--source include/rpl_init.inc
+
+--connection server_1
+-- source include/have_innodb.inc
+--connection server_2
+-- source include/have_innodb.inc
+--connection server_3
+-- source include/have_innodb.inc
+--connection server_1
+
+--echo #
+--echo # Test Case 1: An XA transaction without any statements should not be
+--echo # binlogged
+--let $trx_statements=
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+
+--echo #
+--echo # Test Case 2: An XA transaction consisting of a successfully rolled back
+--echo # statement should not be binlogged
+--let $trx_statements= T
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+
+--echo #
+--echo # Test Case 3: An XA transaction with a statement that cannot be rolled
+--echo # back should be binlogged
+
+# TODO: remove work-around MDEV-24654 when fixed.
+--connection server_1
+set @sav_binlog_format = @@binlog_format;
+set @@binlog_format = row;
+set @@global.binlog_format = row;
+--let $trx_statements= N
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+--connection server_1
+set @@binlog_format = @sav_binlog_format;
+set @@global.binlog_format = @sav_binlog_format;
+
+
+--echo #
+--echo # Test Case 4: An XA transaction with multiple statements that can all
+--echo # be rolled back should not be binlogged
+--let $trx_statements= T,T
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+
+--echo #
+--echo # Test Case 5: A mixed XA transaction consisting of one statement that
+--echo # can successfully be rolled back (first statement), and another that
+--echo # can not (second statement) should be binlogged
+
+--connection server_1
+set @sav_binlog_format = @@binlog_format;
+set @@binlog_format = row;
+set @@global.binlog_format = row;
+--let $trx_statements= T,N
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+--connection server_1
+set @@binlog_format = @sav_binlog_format;
+set @@global.binlog_format = @sav_binlog_format;
+
+
+--echo #
+--echo # Test Case 6: A mixed XA transaction consisting of one statement that
+--echo # cannot successfully be rolled back (first statement), and another that
+--echo # can (second statement) should be binlogged
+
+--connection server_1
+set @sav_binlog_format = @@binlog_format;
+set @@binlog_format = row;
+set @@global.binlog_format = row;
+--let $trx_statements= N,T
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+--connection server_1
+set @@binlog_format = @sav_binlog_format;
+set @@global.binlog_format = @sav_binlog_format;
+
+--echo #
+--echo # Test Case 7: An XA transaction consisting of two failed
+--echo # non-transactional statements should be binlogged
+
+--connection server_1
+set @sav_binlog_format = @@binlog_format;
+set @@binlog_format = row;
+set @@global.binlog_format = row;
+--let $trx_statements= N,N
+
+--let $xa_completion_action= COMMIT
+--source include/rpl_xa_empty_transaction.inc
+
+--let $xa_completion_action= ROLLBACK
+--source include/rpl_xa_empty_transaction.inc
+
+--connection server_1
+set @@binlog_format = @sav_binlog_format;
+set @@global.binlog_format = @sav_binlog_format;
+
+#
+# Cleanup
+--connection server_1
+--source include/rpl_end.inc
+
+--echo # End of rpl_xa_empty_transaction.test