MDEV-9510 Segmentation fault in binlog thread causes crash

With combination of --log-bin and Galera the server may crash
reporting two characteristic stacks:

  /usr/sbin/mysqld(_ZN13MYSQL_BIN_LOG13mark_xid_doneEmb+0xc7)[0x7f182a8e2cb7]
  /usr/sbin/mysqld(binlog_background_thread+0x2b5)[0x7f182a8e3275]

or

  /usr/sbin/mysqld(_ZN13MYSQL_BIN_LOG21do_checkpoint_requestEm+0x9d)[0x7ff395b2dafd]
  /usr/sbin/mysqld(_ZN13MYSQL_BIN_LOG20checkpoint_and_purgeEm+0x11)[0x7ff395b2db91]
  /usr/sbin/mysqld(_ZN13MYSQL_BIN_LOG16rotate_and_purgeEb+0xc2)[0x7ff395b300b2]

The reason of the failure appears to be non-matching decrements for
  `xid_count_per_binlog::xid_count`
which can occur when a transaction is executed having its connection issued
`SET @@sql_log_bin=0`. In such case the xid count is not incremented but
its decrements still runs to turn `binlog_xid_count_list` into improper state
which the following FLUSH BINARY LOGS exposes through the crash.

*Note_1*: the regression test reuses an existing galera.sql_log_bin
which does not run stably (even in its base form) by mtr with --log-bin.

*Note_2*: 10.0-galera branch is free of this issue having missed MDEV-7205
fixes.

mysql-test/suite/galera/r/sql_log_bin.result

@@ -6,6 +6,7 @@ INSERT INTO t1 VALUES (1);
 # Disable binary logging for current session
 SET SQL_LOG_BIN=OFF;
 INSERT INTO t1 VALUES (2);
+FLUSH BINARY LOGS;
 CREATE TABLE t2(c1 INT PRIMARY KEY) ENGINE=INNODB;
 INSERT INTO t2 VALUES (1);
 CREATE TABLE test.t3 AS SELECT * from t1;

mysql-test/suite/galera/t/sql_log_bin.test

@@ -1,5 +1,13 @@
 # Test to check the behavior of galera cluster with sql_log_bin=ON|OFF & binary
 # logging is disabled. sql_bin_log should not affect galera replication.
+#
+# The following bugfixes are tested:
+#
+# MDEV-9510: Segmentation fault in binlog thread.
+#   A scenario otherwise causing a similar segfault is replayed.
+#   The test must pass having no crashes.
+#   The sequence of sql statements is provided by original
+#   sql_log_bin.test augmented with a FLUSH BINLOG LOGS, below.
 
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
@@ -15,6 +23,10 @@ INSERT INTO t1 VALUES (1);
 --echo # Disable binary logging for current session
 SET SQL_LOG_BIN=OFF;
 INSERT INTO t1 VALUES (2);
+
+# MDEV-9510: the following binlog rotation due to FLUSH segfaults wo/  the fixes
+FLUSH BINARY LOGS;
+
 CREATE TABLE t2(c1 INT PRIMARY KEY) ENGINE=INNODB;
 INSERT INTO t2 VALUES (1);
 CREATE TABLE test.t3 AS SELECT * from t1;

sql/log.cc

@@ -7165,8 +7165,15 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd,
     mode. Also, do not write the cached updates to binlog if binary logging is
     disabled (log-bin/sql_log_bin).
   */
-  if (wsrep_emulate_bin_log || !(thd->variables.option_bits & OPTION_BIN_LOG))
+  if (wsrep_emulate_bin_log)
+  {
     DBUG_RETURN(0);
+  }
+  else if (!(thd->variables.option_bits & OPTION_BIN_LOG))
+  {
+    cache_mngr->need_unlog= false;
+    DBUG_RETURN(0);
+  }
 
   entry.thd= thd;
   entry.cache_mngr= cache_mngr;
@@ -9489,11 +9496,19 @@ TC_LOG_BINLOG::log_and_order(THD *thd, my_xid xid, bool all,
 
   if (err)
     DBUG_RETURN(0);
+
+  bool need_unlog= cache_mngr->need_unlog;
+  /*
+    The transaction won't need the flag anymore.
+    Todo/fixme: consider to move the statement into cache_mngr->reset()
+                relocated to the current or later point.
+  */
+  cache_mngr->need_unlog= false;
   /*
     If using explicit user XA, we will not have XID. We must still return a
     non-zero cookie (as zero cookie signals error).
   */
-  if (!xid || !cache_mngr->need_unlog)
+  if (!xid || !need_unlog)
     DBUG_RETURN(BINLOG_COOKIE_DUMMY(cache_mngr->delayed_error));
   else
     DBUG_RETURN(BINLOG_COOKIE_MAKE(cache_mngr->binlog_id,
@@ -9566,6 +9581,9 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint)
     if (b->binlog_id == binlog_id)
     {
       --b->xid_count;
+
+      DBUG_ASSERT(b->xid_count >= 0); // catch unmatched (++) decrement
+
       break;
     }
     first= false;