MDEV-39958 Galera support for --binlog-storage-engine

phase 1 of the development Galera did not support the binlog-in-server implementation, server refused to start with the combination. This patch is phase 1 work to make the basic replication to work and removes the startup check fpr rejecting binlog-in-engine use in cluster. There is one limitation: a transaction larger than the binlog cache can not be replicated yet. New mtr test galera.galera_binlog_in_engine runs a 2-node cluster with --binlog-storage-engine=innodb, node_2 joining through rsync SST.
2026-10-23 18:49:29 +03:00 · 2026-10-09 17:34:55 +03:00
parent 9d7ab2c009
commit eac3e3ef6b
11 changed files with 452 additions and 8 deletions
--- a/mysql-test/suite/galera/r/galera_binlog_in_engine.result
+++ b/mysql-test/suite/galera/r/galera_binlog_in_engine.result
@@ -0,0 +1,104 @@
+connection node_2;
+connection node_1;
+connection node_1;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+call mtr.add_suppression("WSREP: Transaction data was spilled to the binlog storage engine before it was replicated");
+SELECT @@binlog_storage_engine;
+@@binlog_storage_engine
+innodb
+connection node_2;
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@binlog_storage_engine;
+@@binlog_storage_engine
+innodb
+#
+# Autocommit DML and DDL
+#
+connection node_1;
+CREATE TABLE t1 (id INT PRIMARY KEY, b VARCHAR(1000)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 'a');
+UPDATE t1 SET b = 'b' WHERE id = 1;
+INSERT INTO t1 VALUES (2, 'c');
+DELETE FROM t1 WHERE id = 2;
+connection node_2;
+SELECT * FROM t1;
+id	b
+1	b
+#
+# Multi-statement transaction with a savepoint rollback
+#
+connection node_1;
+START TRANSACTION;
+INSERT INTO t1 VALUES (10, 'x');
+SAVEPOINT s1;
+INSERT INTO t1 VALUES (11, 'rolled back');
+ROLLBACK TO SAVEPOINT s1;
+INSERT INTO t1 VALUES (12, 'y');
+COMMIT;
+# Statement rollback inside a transaction
+START TRANSACTION;
+INSERT INTO t1 VALUES (20, 'z');
+INSERT INTO t1 VALUES (21, 'rolled back'), (20, 'dup');
+ERROR 23000: Duplicate entry '20' for key 'PRIMARY'
+COMMIT;
+connection node_2;
+SELECT * FROM t1 ORDER BY id;
+id	b
+1	b
+10	x
+12	y
+20	z
+#
+# Writes on node_2 replicate to node_1
+#
+connection node_2;
+INSERT INTO t1 VALUES (30, 'from node_2');
+connection node_1;
+SELECT * FROM t1 WHERE id = 30;
+id	b
+30	from node_2
+#
+# A transaction smaller than the binlog cache but with several
+# events replicates.
+#
+connection node_1;
+START TRANSACTION;
+COMMIT;
+connection node_2;
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 100 AND 119;
+COUNT(*)
+20
+#
+# A transaction larger than the binlog cache is spilled to the
+# engine binlog before it is replicated. This is not yet supported,
+# the transaction is rolled back and nothing is replicated.
+#
+connection node_1;
+START TRANSACTION;
+COMMIT;
+ERROR HY000: Galera replication of a transaction larger than the binlog cache is not yet supported with --binlog-storage-engine
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 200 AND 259;
+COUNT(*)
+0
+# The cluster is still consistent and operational
+INSERT INTO t1 VALUES (300, 'after large');
+connection node_2;
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 200 AND 259;
+COUNT(*)
+0
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+26
+connection node_1;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+26
+connection node_1;
+DROP TABLE t1;
+#
+# Both nodes have binlogged the same replicated transactions with the
+# same GTIDs (wsrep_gtid_domain_id=1) in the engine binlog
+#
+connection node_2;
+connection node_1;
+# gtid_binlog_pos domain 1 matches on both nodes
--- a/mysql-test/suite/galera/t/galera_binlog_in_engine.cnf
+++ b/mysql-test/suite/galera/t/galera_binlog_in_engine.cnf
@@ -0,0 +1,19 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+binlog-cache-size=32768
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=11
+
+[mysqld.2]
+log-bin
+log-slave-updates
+binlog-storage-engine=innodb
+binlog-cache-size=32768
+wsrep_gtid_mode=ON
+wsrep_gtid_domain_id=1
+server-id=12
--- a/mysql-test/suite/galera/t/galera_binlog_in_engine.test
+++ b/mysql-test/suite/galera/t/galera_binlog_in_engine.test
@@ -0,0 +1,140 @@
+#
+# Galera with the binlog implemented in InnoDB (--binlog-storage-engine=innodb).
+#
+# Both nodes write the binlog into InnoDB, the applier on node_2 binlogs the
+# replicated transactions through --log-slave-updates. Node_2 joins the
+# cluster through rsync SST, which must transfer the InnoDB binlog files.
+#
+--source include/galera_cluster.inc
+--source include/have_innodb_binlog.inc
+
+--connection node_1
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+call mtr.add_suppression("WSREP: Transaction data was spilled to the binlog storage engine before it was replicated");
+SELECT @@binlog_storage_engine;
+--connection node_2
+call mtr.add_suppression("Galera support for --binlog-storage-engine is experimental");
+SELECT @@binlog_storage_engine;
+
+--echo #
+--echo # Autocommit DML and DDL
+--echo #
+--connection node_1
+CREATE TABLE t1 (id INT PRIMARY KEY, b VARCHAR(1000)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 'a');
+UPDATE t1 SET b = 'b' WHERE id = 1;
+INSERT INTO t1 VALUES (2, 'c');
+DELETE FROM t1 WHERE id = 2;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1 WHERE b = 'b'
+--source include/wait_condition.inc
+SELECT * FROM t1;
+
+--echo #
+--echo # Multi-statement transaction with a savepoint rollback
+--echo #
+--connection node_1
+START TRANSACTION;
+INSERT INTO t1 VALUES (10, 'x');
+SAVEPOINT s1;
+INSERT INTO t1 VALUES (11, 'rolled back');
+ROLLBACK TO SAVEPOINT s1;
+INSERT INTO t1 VALUES (12, 'y');
+COMMIT;
+
+--echo # Statement rollback inside a transaction
+START TRANSACTION;
+INSERT INTO t1 VALUES (20, 'z');
+--error ER_DUP_ENTRY
+INSERT INTO t1 VALUES (21, 'rolled back'), (20, 'dup');
+COMMIT;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 4 FROM t1
+--source include/wait_condition.inc
+SELECT * FROM t1 ORDER BY id;
+
+--echo #
+--echo # Writes on node_2 replicate to node_1
+--echo #
+--connection node_2
+INSERT INTO t1 VALUES (30, 'from node_2');
+
+--connection node_1
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1 WHERE id = 30
+--source include/wait_condition.inc
+SELECT * FROM t1 WHERE id = 30;
+
+--echo #
+--echo # A transaction smaller than the binlog cache but with several
+--echo # events replicates.
+--echo #
+--connection node_1
+START TRANSACTION;
+--disable_query_log
+--let $i = 100
+while ($i < 120)
+{
+  --eval INSERT INTO t1 VALUES ($i, REPEAT('s', 1000))
+  --inc $i
+}
+--enable_query_log
+COMMIT;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 20 FROM t1 WHERE id BETWEEN 100 AND 119
+--source include/wait_condition.inc
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 100 AND 119;
+
+--echo #
+--echo # A transaction larger than the binlog cache is spilled to the
+--echo # engine binlog before it is replicated. This is not yet supported,
+--echo # the transaction is rolled back and nothing is replicated.
+--echo #
+--connection node_1
+START TRANSACTION;
+--disable_query_log
+--let $i = 200
+while ($i < 260)
+{
+  --eval INSERT INTO t1 VALUES ($i, REPEAT('l', 1000))
+  --inc $i
+}
+--enable_query_log
+--error ER_NOT_YET_SUPPORTED_ENGINE_BINLOG
+COMMIT;
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 200 AND 259;
+
+--echo # The cluster is still consistent and operational
+INSERT INTO t1 VALUES (300, 'after large');
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1 WHERE id = 300
+--source include/wait_condition.inc
+SELECT COUNT(*) FROM t1 WHERE id BETWEEN 200 AND 259;
+SELECT COUNT(*) FROM t1;
+--connection node_1
+SELECT COUNT(*) FROM t1;
+
+--connection node_1
+DROP TABLE t1;
+
+--echo #
+--echo # Both nodes have binlogged the same replicated transactions with the
+--echo # same GTIDs (wsrep_gtid_domain_id=1) in the engine binlog
+--echo #
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 0 FROM information_schema.tables WHERE table_name = 't1'
+--source include/wait_condition.inc
+--let $gtid_2 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(^|,)1-[0-9]+-[0-9]+')`
+--connection node_1
+--let $gtid_1 = `SELECT REGEXP_SUBSTR(@@gtid_binlog_pos, '(^|,)1-[0-9]+-[0-9]+')`
+if (!$gtid_1)
+{
+  --die No replicated transactions in gtid_binlog_pos of node_1
+}
+if ($gtid_1 != $gtid_2)
+{
+  --die gtid_binlog_pos domain 1 differs between the nodes: $gtid_1 vs $gtid_2
+}
+--echo # gtid_binlog_pos domain 1 matches on both nodes
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -484,7 +484,9 @@ EOF
 #         --exclude grastate.txt --exclude '*.pem' \
 #         --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index')

-# New filter - exclude everything except dirs (schemas) and innodb files
+# New filter - exclude everything except dirs (schemas) and innodb files.
+# The InnoDB binlog files (--binlog-storage-engine=innodb) are transferred
+# with the rest of the InnoDB state, as the InnoDB redo log refers to them.
 FILTER="-f '- /lost+found'
        -f '- /.zfs'
        -f '- /.fseventsd'
@@ -499,6 +501,7 @@ FILTER="-f '- /lost+found'
        -f '- $ib_log_dir/ib_logfile[0-9]*'
        -f '- $ar_log_dir/aria_log_control'
        -f '- $ar_log_dir/aria_log.*'
+        -f '+ /binlog-[0-9]*.ibb'
        -f '+ /*/'
        -f '- /*'"

--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -2006,7 +2006,18 @@ int ha_commit_trans(THD *thd, bool all)
      continue;
    /* We do not need to 2pc the binlog with the engine that implements it. */
    if (ht == opt_binlog_engine_hton)
+    {
+#ifdef WITH_WSREP
+      /*
+        with binlog-in-engine, innodb prepare is skipped, but wsrep prepare should be
+        done, we will do it here
+      */
+      if (run_wsrep_hooks && ht->flags & HTON_WSREP_REPLICATION &&
+          (wsrep_before_prepare(thd, all) || wsrep_after_prepare(thd, all)))
+        goto err;
+#endif /* WITH_WSREP */
      continue;
+    }
    /*
      Sic: we know that prepare() is not NULL since otherwise
      trans->no_2pc would have been set.
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -5423,11 +5423,9 @@ static int init_server_components()
  if (WSREP_ON && !wsrep_recovery && !opt_abort)
  {
    if (binlog_engine_used)
-    {
-      sql_print_error("Galera cannot be used with the "
-                      "--binlog-storage-engine option");
-      unireg_abort(1);
-    }
+      sql_print_warning("Galera support for --binlog-storage-engine is "
+                        "experimental: replicating a transaction larger "
+                        "than the binlog cache is not yet supported");
    if (opt_bootstrap) // bootstrap option given - disable wsrep functionality
    {
      wsrep_provider_init(WSREP_NONE);
--- a/sql/sql_reload.cc
+++ b/sql/sql_reload.cc
@@ -188,8 +188,8 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options,
          *write_to_binlog= -1;

        /* Note that WSREP(thd) might not be true here e.g. during
-        SST. */
-        if (WSREP_ON)
+        SST. The binlog-in-engine does not use checkpoint events. */
+        if (WSREP_ON && !opt_binlog_engine_hton)
        {
          /* Wait for last binlog checkpoint event to be logged. */
          mysql_bin_log.wait_for_last_checkpoint_event();
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -25,6 +25,82 @@
 #include "transaction.h"

 extern handlerton *binlog_hton;
+
+static void wsrep_report_engine_binlog_spill(my_off_t log_position,
+                                             my_off_t spilled)
+{
+  WSREP_WARN("Transaction data was spilled to the binlog storage engine "
+             "before it was replicated (replicated up to %llu, "
+             "spilled up to %llu). Replicating transactions larger than "
+             "the binlog cache is not yet supported with "
+             "--binlog-storage-engine.",
+             (ulonglong) log_position, (ulonglong) spilled);
+  my_error(ER_NOT_YET_SUPPORTED_ENGINE_BINLOG, MYF(0),
+           "Galera replication of a transaction larger than the binlog cache");
+}
+
+/*
+  Get the binlog cache contents from log_position to the end of the cache,
+  when the binlog is implemented in a storage engine
+  (--binlog-storage-engine).
+
+  Such a cache has no temporary file. When it fills up, its contents are
+  spilled into the engine binlog as out-of-band data and are no longer
+  available in the cache. Also, the cache must not be switched to
+  READ_CACHE and back with reinit_io_cache(), as that would replace the
+  write function which does the spilling. So the data is taken directly
+  from the in-memory buffer.
+
+  @param cache         binlog cache, in WRITE_CACHE mode
+  @param log_position  position to start from
+  @param data          set to the start of the data
+  @param len           set to the length of the data
+
+  @return 0 on success, 1 if part of the data has already been spilled
+          into the engine binlog
+*/
+int wsrep_engine_cache_data(IO_CACHE *cache, my_off_t log_position,
+                            const uchar **data, size_t *len)
+{
+  DBUG_ASSERT(opt_binlog_engine_hton);
+  DBUG_ASSERT(cache->type == WRITE_CACHE);
+  DBUG_ASSERT(log_position <= my_b_tell(cache));
+
+  if (log_position < cache->pos_in_file)
+  {
+    wsrep_report_engine_binlog_spill(log_position, cache->pos_in_file);
+    return 1;
+  }
+
+  *data= cache->request_pos + (log_position - cache->pos_in_file);
+  *len= (size_t) (cache->write_pos - *data);
+  return 0;
+}
+
+bool wsrep_engine_binlog_spilled(THD *thd)
+{
+  if (!opt_binlog_engine_hton ||
+      thd->wsrep_cs().mode() != wsrep::client_state::m_local ||
+      thd->wsrep_trx().state() != wsrep::transaction::s_executing)
+    return false;
+
+  IO_CACHE *trx_cache= wsrep_get_cache(thd, true);
+  IO_CACHE *stmt_cache= wsrep_get_cache(thd, false);
+  my_off_t const log_position= thd->wsrep_sr().log_position();
+
+  if (trx_cache && trx_cache->pos_in_file > log_position)
+  {
+    wsrep_report_engine_binlog_spill(log_position, trx_cache->pos_in_file);
+    return true;
+  }
+  if (stmt_cache && stmt_cache->pos_in_file > 0)
+  {
+    wsrep_report_engine_binlog_spill(0, stmt_cache->pos_in_file);
+    return true;
+  }
+  return false;
+}
+
 /*
  Write the contents of a cache to a memory buffer.

@@ -38,6 +114,31 @@ int wsrep_write_cache_buf(IO_CACHE *cache, uchar **buf, size_t *buf_len)
  my_off_t const saved_pos(my_b_tell(cache));
  DBUG_ENTER("wsrep_write_cache_buf");

+  if (opt_binlog_engine_hton && cache->file < 0)
+  {
+    const uchar *data;
+    size_t length;
+    if (wsrep_engine_cache_data(cache, 0, &data, &length))
+      DBUG_RETURN(ER_ERROR_ON_WRITE);
+    if (length > wsrep_max_ws_size)
+    {
+      WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                 wsrep_max_ws_size, length);
+      DBUG_RETURN(ER_ERROR_ON_WRITE);
+    }
+    if (length)
+    {
+      if (!(*buf= (uchar *) my_malloc(PSI_INSTRUMENT_ME, length, MYF(0))))
+      {
+        WSREP_ERROR("could not allocate buffer: %zu", length);
+        DBUG_RETURN(ER_ERROR_ON_WRITE);
+      }
+      memcpy(*buf, data, length);
+      *buf_len= length;
+    }
+    DBUG_RETURN(0);
+  }
+
  if (reinit_io_cache(cache, READ_CACHE, 0, 0, 0))
  {
    WSREP_ERROR("failed to initialize io-cache");
@@ -124,6 +225,26 @@ static int wsrep_write_cache_inc(THD*      const thd,
  DBUG_ENTER("wsrep_write_cache_inc");
  my_off_t const saved_pos(my_b_tell(cache));

+  if (opt_binlog_engine_hton)
+  {
+    const uchar *data;
+    size_t length;
+    *len= 0;
+    if (wsrep_engine_cache_data(cache, log_position, &data, &length))
+      DBUG_RETURN(1);
+    if (unlikely(length > wsrep_max_ws_size))
+    {
+      WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                 wsrep_max_ws_size, length);
+      DBUG_RETURN(1);
+    }
+    if (length &&
+        thd->wsrep_cs().append_data(wsrep::const_buffer(data, length)))
+      DBUG_RETURN(1);
+    *len= length;
+    DBUG_RETURN(0);
+  }
+
  if (reinit_io_cache(cache, READ_CACHE, log_position, 0, 0))
  {
    WSREP_ERROR("failed to initialize io-cache");
--- a/sql/wsrep_binlog.h
+++ b/sql/wsrep_binlog.h
@@ -22,6 +22,26 @@
 #define HEAP_PAGE_SIZE 65536 /* 64K */
 #define WSREP_MAX_WS_SIZE 2147483647 /* 2GB */

+/*
+  Get the binlog cache contents from log_position to the end of the cache,
+  when the binlog is implemented in a storage engine
+  (--binlog-storage-engine). The data is not copied.
+
+  @return 0 on success, 1 if part of the data has already been spilled
+          into the engine binlog
+ */
+int wsrep_engine_cache_data(IO_CACHE *cache, my_off_t log_position,
+                            const uchar **data, size_t *len);
+
+/*
+  Check, before certification of a local transaction, whether data not yet
+  replicated has been spilled into the binlog storage engine
+  (--binlog-storage-engine), so that it can not be replicated.
+
+  @return true, with the error set, if data has been spilled
+ */
+bool wsrep_engine_binlog_spilled(THD *thd);
+
 /*
  Write the contents of a cache to a memory buffer.

--- a/sql/wsrep_client_service.cc
+++ b/sql/wsrep_client_service.cc
@@ -168,6 +168,26 @@ int Wsrep_client_service::prepare_fragment_for_replication(
  }

  const my_off_t saved_pos(my_b_tell(cache));
+
+  if (opt_binlog_engine_hton)
+  {
+    const uchar *data;
+    size_t length;
+    if (wsrep_engine_cache_data(cache, thd->wsrep_sr().log_position(),
+                                &data, &length))
+      DBUG_RETURN(1);
+    if (length > wsrep_max_ws_size)
+    {
+      WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                 wsrep_max_ws_size, length);
+      DBUG_RETURN(1);
+    }
+    buffer.push_back(reinterpret_cast<const char*>(data),
+                     reinterpret_cast<const char*>(data + length));
+    log_position= saved_pos;
+    DBUG_RETURN(0);
+  }
+
  if (reinit_io_cache(cache, READ_CACHE, thd->wsrep_sr().log_position(), 0, 0))
  {
    DBUG_RETURN(1);
--- a/sql/wsrep_trans_observer.h
+++ b/sql/wsrep_trans_observer.h
@@ -262,6 +262,10 @@ static inline int wsrep_before_prepare(THD* thd, bool all)
  WSREP_DEBUG("wsrep_before_prepare: %d", wsrep_is_real(thd, all));
  int ret= 0;
  DBUG_ASSERT(wsrep_run_commit_hook(thd, all));
+  if (wsrep_engine_binlog_spilled(thd))
+  {
+    DBUG_RETURN(1);
+  }
  if ((ret= thd->wsrep_parallel_slave_wait_for_prior_commit()))
  {
    DBUG_RETURN(ret);
@@ -320,6 +324,10 @@ static inline int wsrep_before_commit(THD* thd, bool all)
  THD_STAGE_INFO(thd, stage_waiting_certification);
  int ret= 0;
  DBUG_ASSERT(wsrep_run_commit_hook(thd, all));
+  if (wsrep_engine_binlog_spilled(thd))
+  {
+    DBUG_RETURN(1);
+  }

  wsrep::provider::seq_cb_t seq_cb{
      thd, wsrep_parallel_slave_wakeup_subsequent_commits};