MDEV-34705: Binlog-in-engine: Configurable binlog directory

Add option --binlog-directory, used to place the binlogs outside the data
directory (eg. to put them on different disk/file system).

Disallow specifying the binlog name in --log-bin when
--binlog-storage-engine is used, as the name is then not user configurable.

A ToDo (not implemented in this commit) is to use the --binlog-directory
value, if given, also for the legacy binlog implementation.

Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>

mysql-test/include/have_innodb_binlog.opt

@@ -1 +1 @@
---binlog-storage-engine=innodb --max-binlog-size=256K
+--log-bin --binlog-storage-engine=innodb --max-binlog-size=256K

mysql-test/suite/binlog_in_engine/binlog_in_engine.test

@@ -1,5 +1,10 @@
---source include/have_innodb_binlog.inc
 --source include/have_binlog_format_mixed.inc
+# ToDo: For now, this has to come _after_ have_log_bin.inc (or
+# have_binlog_format_*.inc), to override --log-bin=master-bin with empty
+# --log-bin, as engine does not allow to set the binlog name.
+# Alternatively, maybe could have separate have_innodb_binlog_format_*.inc
+# files and only need to include the one.
+--source include/have_innodb_binlog.inc
 
 CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
 --let $gtid_pos= `SELECT @@last_gtid`

mysql-test/suite/binlog_in_engine/binlog_in_engine2.test

@@ -1,5 +1,5 @@
---source include/have_innodb_binlog.inc
 --source include/have_binlog_format_mixed.inc
+--source include/have_innodb_binlog.inc
 
 CREATE TABLE sbtest1(
   id INTEGER NOT NULL AUTO_INCREMENT,

mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.opt

@@ -1 +1,2 @@
 --innodb-binlog-state-interval=65536
+--binlog-directory=binlogs

mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test

@@ -1,8 +1,11 @@
 --source include/big_test.inc
---source include/have_innodb_binlog.inc
 --source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
 --source include/have_sequence.inc
 
+# Note: This test also tests the --binlog-directory option by putting it
+# in binlog_in_engine_restart.opt .
+
 CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1, 0);
 let $i= 0;

mysql-test/suite/binlog_in_engine/rpl_gtid_index-master.opt

@@ -1,2 +1,3 @@
 --max-binlog-size=128k
 --innodb-binlog-state-interval=64k
+--log-bin

mysql-test/suite/binlog_in_engine/rpl_gtid_index-slave.opt

@@ -0,0 +1 @@
+--log-bin

mysql-test/suite/binlog_in_engine/rpl_oob.test

@@ -1,6 +1,6 @@
---source include/have_innodb_binlog.inc
 --source include/have_binlog_format_row.inc
 --source include/master-slave.inc
+--source include/have_innodb_binlog.inc
 
 # Test a number of transactions that are large and get interleaved with each
 # other over multiple binlog files.

sql/handler.h

@@ -1534,7 +1534,7 @@ struct handlerton
   void *optimizer_costs;                        /* Costs are stored here */
 
   /* Optional implementation of binlog in the engine. */
-  bool (*binlog_init)(size_t binlog_size);
+  bool (*binlog_init)(size_t binlog_size, const char *directory);
   /* Binlog an event group that doesn't go through commit_ordered. */
   bool (*binlog_write_direct)(IO_CACHE *cache,
                               handler_binlog_event_group_info *binlog_info,

sql/mysqld.cc

@@ -364,8 +364,10 @@ char server_uid[SERVER_UID_SIZE+1];   // server uid will be written here
 /* Global variables */
 
 bool opt_bin_log, opt_bin_log_used=0, opt_ignore_builtin_innodb= 0;
+static bool opt_bin_log_nonempty, opt_bin_log_path;
 char *opt_binlog_storage_engine= const_cast<char *>("");
 static plugin_ref opt_binlog_engine_plugin;
+char *opt_binlog_directory;
 handlerton *opt_binlog_engine_hton;
 bool opt_bin_log_compress;
 uint opt_bin_log_compress_min_len;
@@ -5062,6 +5064,7 @@ static int init_server_components()
 
     char buf[FN_REFLEN];
     const char *ln;
+    /* ToDo: Here we also need to add in opt_binlog_directory, if given. */
     ln= mysql_bin_log.generate_name(opt_bin_logname, "-bin", 1, buf);
     if (!opt_bin_logname[0] && !opt_binlog_index_name)
     {
@@ -5460,6 +5463,14 @@ static int init_server_components()
   if (init_gtid_pos_auto_engines())
     unireg_abort(1);
 
+  if (opt_binlog_directory && opt_binlog_directory[0] &&
+      opt_bin_log_path)
+  {
+    sql_print_error("Cannot specify a directory path for the binlog in "
+                    "--log-bin when --binlog-directory-path is also used");
+    unireg_abort(1);
+  }
+
   if (opt_binlog_storage_engine && *opt_binlog_storage_engine && !opt_bootstrap)
   {
     LEX_CSTRING name= { opt_binlog_storage_engine, strlen(opt_binlog_storage_engine) };
@@ -5472,17 +5483,26 @@ static int init_server_components()
         sql_print_error("Unknown/unsupported storage engine: %s",
                         opt_binlog_storage_engine);
       else
-        sql_print_error("Engine %s is not available for --innodb-binlog-engine",
+        sql_print_error("Engine %s is not available for "
+                        "--binlog-storage-engine",
                         opt_binlog_storage_engine);
       unireg_abort(1);
     }
     if (!opt_binlog_engine_hton->binlog_write_direct ||
         !opt_binlog_engine_hton->get_binlog_reader)
     {
-      sql_print_error("Engine %s does not support --innodb-binlog-engine",
+      sql_print_error("Engine %s does not support --binlog-storage-engine",
                       opt_binlog_storage_engine);
       unireg_abort(1);
     }
+
+    if (opt_bin_log_nonempty)
+    {
+      sql_print_error("Binlog name can not be set with --log-bin when "
+                      "--binlog-storage-engine is used. Use --binlog-directory "
+                      "to specify a separate directory for binlogs");
+      unireg_abort(1);
+    }
   }
 
 #ifdef USE_ARIA_FOR_TMP_TABLES
@@ -5537,15 +5557,20 @@ static int init_server_components()
 
   if (opt_bin_log)
   {
+    mysql_mutex_t *log_lock= mysql_bin_log.get_log_lock();
     int error;
     if (opt_binlog_engine_hton)
     {
-      if ((*opt_binlog_engine_hton->binlog_init)((size_t)max_binlog_size))
+      mysql_mutex_lock(log_lock);
+      if ((*opt_binlog_engine_hton->binlog_init)((size_t)max_binlog_size,
+                                                 opt_binlog_directory))
         error= 1;
+      mysql_mutex_unlock(log_lock);
+      if (unlikely(error))
+        unireg_abort(1);
     }
     if (true) /* ToDo: `else` branch (don't open legacy binlog if using engine implementation). */
     {
-      mysql_mutex_t *log_lock= mysql_bin_log.get_log_lock();
       mysql_mutex_lock(log_lock);
       error= mysql_bin_log.open(opt_bin_logname, 0, 0,
                                 WRITE_CACHE, max_binlog_size, 0, TRUE);
@@ -8255,6 +8280,9 @@ mysqld_get_one_option(const struct my_option *opt, const char *argument,
   case (int) OPT_BIN_LOG:
     opt_bin_log= MY_TEST(argument != disabled_my_option);
     opt_bin_log_used= 1;
+    opt_bin_log_nonempty= (argument && argument[0]);
+    opt_bin_log_path= argument &&
+      (strchr(argument, FN_LIBCHAR) || strchr(argument, FN_LIBCHAR2));
     break;
   case (int) OPT_LOG_BASENAME:
   {

sql/mysqld.h

@@ -106,6 +106,7 @@ uint temp_pool_set_next();
 extern bool opt_large_files;
 extern bool opt_update_log, opt_bin_log, opt_error_log, opt_bin_log_compress;
 extern char *opt_binlog_storage_engine;
+extern char *opt_binlog_directory;
 extern handlerton *opt_binlog_engine_hton;
 extern uint opt_bin_log_compress_min_len;
 extern my_bool opt_log, opt_bootstrap;

sql/sys_vars.cc

@@ -1813,6 +1813,16 @@ Sys_max_binlog_size(
        BLOCK_SIZE(IO_SIZE), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
        ON_UPDATE(fix_max_binlog_size));
 
+
+static Sys_var_charptr_fscs Sys_binlog_directory(
+       "binlog_directory",
+       "Directory path (absolute or relative to datadir) where binlog files "
+       "are stored. If this is used, must not specify a directory path for "
+       "--log-bin",
+       READ_ONLY GLOBAL_VAR(opt_binlog_directory), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(0));
+
+
 static bool fix_max_connections(sys_var *self, THD *thd, enum_var_type type)
 {
   return false;

storage/innobase/fsp/fsp_binlog.cc

@@ -166,7 +166,7 @@ fsp_binlog_open(const char *file_name, pfs_os_file_t fh,
   const uint32_t page_size= (uint32_t)srv_page_size;
   const uint32_t page_size_shift= srv_page_size_shift;
 
-  os_offset_t binlog_size= max_binlog_size;
+  os_offset_t binlog_size= innodb_binlog_size_in_pages << srv_page_size_shift;
   if (open_empty && file_size < binlog_size) {
     /*
       A crash may have left a partially pre-allocated file. If so, extend it
@@ -268,7 +268,7 @@ dberr_t fsp_binlog_tablespace_create(uint64_t file_no, fil_space_t **new_space)
 	if(srv_read_only_mode)
 		return DB_ERROR;
 
-        char name[BINLOG_NAME_LEN];
+        char name[OS_FILE_MAX_PATH];
         binlog_name_make(name, file_no);
 
 	os_file_create_subdirs_if_needed(name);
@@ -620,7 +620,7 @@ binlog_chunk_reader::fetch_current_page()
     /* Tablespace is not open, just read from the file. */
     if (cur_file_handle < (File)0)
     {
-      char filename[BINLOG_NAME_LEN];
+      char filename[OS_FILE_MAX_PATH];
       MY_STAT stat_buf;
 
       binlog_name_make(filename, s.file_no);

storage/innobase/handler/innodb_binlog.cc

@@ -33,6 +33,7 @@ InnoDB implementation of binlog.
 
 
 uint32_t innodb_binlog_size_in_pages;
+const char *innodb_binlog_directory;
 
 /* Current write position in active binlog file. */
 uint32_t binlog_cur_page_no;
@@ -434,7 +435,7 @@ innodb_binlog_startup_init()
   use the innodb implementation (with --binlog-storage-engine=innodb).
 */
 bool
-innodb_binlog_init(size_t binlog_size)
+innodb_binlog_init(size_t binlog_size, const char *directory)
 {
   uint64_t pages= binlog_size >> srv_page_size_shift;
   if (UNIV_LIKELY(pages > (uint64_t)UINT32_MAX)) {
@@ -450,6 +451,16 @@ innodb_binlog_init(size_t binlog_size)
   }
   innodb_binlog_size_in_pages= (uint32_t)pages;
 
+  if (!directory || !directory[0])
+    directory= ".";
+  else if (strlen(directory) + BINLOG_NAME_MAX_LEN > OS_FILE_MAX_PATH)
+  {
+    ib::error() << "Specified binlog directory path '" << directory <<
+      "' is too long.";
+    return true;
+  }
+  innodb_binlog_directory= directory;
+
   first_open_binlog_file_no= ~(uint64_t)0;
   binlog_cur_end_offset[0].store(~(uint64_t)0, std::memory_order_relaxed);
   binlog_cur_end_offset[1].store(~(uint64_t)0, std::memory_order_relaxed);
@@ -537,7 +548,7 @@ find_pos_in_binlog(uint64_t file_no, size_t file_size, byte *page_buf,
   const uint32_t page_size= (uint32_t)srv_page_size;
   const uint32_t page_size_shift= (uint32_t)srv_page_size_shift;
   const uint32_t idx= file_no & 1;
-  char file_name[BINLOG_NAME_LEN];
+  char file_name[OS_FILE_MAX_PATH];
   uint32_t p_0, p_1, p_2, last_nonempty;
   dberr_t err;
   byte *p, *page_end;
@@ -643,9 +654,15 @@ innodb_binlog_discover()
   uint64_t file_no;
   const uint32_t page_size= (uint32_t)srv_page_size;
   const uint32_t page_size_shift= (uint32_t)srv_page_size_shift;
-  MY_DIR *dir= my_dir(".", MYF(MY_WME|MY_WANT_STAT));  // ToDo: configurable binlog directory, and don't ask my_dir to stat every file found
+  MY_DIR *dir= my_dir(innodb_binlog_directory, MYF(MY_WANT_STAT));
   if (!dir)
+  {
+    if (my_errno == ENOENT)
+      return 0;
+    ib::error() << "Could not read the binlog directory '" <<
+      innodb_binlog_directory << "', error code " << my_errno << ".";
     return -1;
+  }
 
   struct found_binlogs UNINIT_VAR(binlog_files);
   binlog_files.found_binlogs= 0;
@@ -1110,7 +1127,7 @@ binlog_state_recover()
   state.init();
   uint64_t diff_state_interval= 0;
   uint32_t page_no= 0;
-  char filename[BINLOG_NAME_LEN];
+  char filename[OS_FILE_MAX_PATH];
 
   binlog_name_make(filename,
                    active_binlog_file_no.load(std::memory_order_relaxed));
@@ -1909,7 +1926,7 @@ gtid_search::read_gtid_state_file_no(rpl_binlog_state_base *state,
       }
       if (cur_open_file < (File)0)
       {
-        char filename[BINLOG_NAME_LEN];
+        char filename[OS_FILE_MAX_PATH];
         binlog_name_make(filename, file_no);
         cur_open_file= my_open(filename, O_RDONLY | O_BINARY, MYF(0));
         if (cur_open_file < (File)0)

storage/innobase/include/innodb_binlog.h

@@ -62,25 +62,29 @@ struct chunk_data_base {
 
 #define BINLOG_NAME_BASE "binlog-"
 #define BINLOG_NAME_EXT ".ibb"
-/* '.' + '/' + "binlog-" + (<=20 digits) + '.' + "ibb" + '\0'. */
-#define BINLOG_NAME_LEN 1 + 1 + 7 + 20 + 1 + 3 + 1
-static inline void
-binlog_name_make(char name_buf[BINLOG_NAME_LEN], uint64_t file_no)
-{
-  sprintf(name_buf, "./" BINLOG_NAME_BASE "%06" PRIu64 BINLOG_NAME_EXT,
-          file_no);
-}
+/* '/' + "binlog-" + (<=20 digits) + '.' + "ibb" + '\0'. */
+#define BINLOG_NAME_MAX_LEN 1 + 1 + 7 + 20 + 1 + 3 + 1
 
 
 extern uint32_t innodb_binlog_size_in_pages;
+extern const char *innodb_binlog_directory;
 extern uint32_t binlog_cur_page_no;
 extern uint32_t binlog_cur_page_offset;
 extern ulonglong innodb_binlog_state_interval;
 extern rpl_binlog_state_base binlog_diff_state;
 
 
+static inline void
+binlog_name_make(char name_buf[OS_FILE_MAX_PATH], uint64_t file_no)
+{
+  snprintf(name_buf, OS_FILE_MAX_PATH,
+           "%s/" BINLOG_NAME_BASE "%06" PRIu64 BINLOG_NAME_EXT,
+           innodb_binlog_directory, file_no);
+}
+
+
 extern void innodb_binlog_startup_init();
-extern bool innodb_binlog_init(size_t binlog_size);
+extern bool innodb_binlog_init(size_t binlog_size, const char *directory);
 extern void innodb_binlog_close();
 extern bool binlog_gtid_state(rpl_binlog_state_base *state, mtr_t *mtr,
                               buf_block_t * &block, uint32_t &page_no,