From 30b5a4de5d169cb82829938ffbe60512be89ffef Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 31 Jul 2014 16:40:32 -0400 Subject: [PATCH] MDEV-6492: MariaDB Galera Cluster cant use rsync sst A donor node does a flush tables and then tries to freeze innodb writes before proceeding with SST. However, innodb_disallow_writes was missing in xtradb. Merged 'InnodbFreeze' patch from maria-5.5-galera's to xtradb. Also, merged some changes missing in innobase's os0file.cc. Added a basic test case for innodb_disallow_writes system variable. --- mysql-test/include/mtr_warnings.sql | 2 - .../r/innodb_disallow_writes_basic.result | 45 +++++++++++++++++++ .../t/innodb_disallow_writes_basic.test | 42 +++++++++++++++++ storage/innobase/os/os0file.cc | 23 +++++++++- storage/xtradb/handler/ha_innodb.cc | 40 +++++++++++++++++ storage/xtradb/include/srv0srv.h | 4 ++ storage/xtradb/os/os0file.cc | 24 +++++++++- storage/xtradb/srv/srv0srv.cc | 25 +++++++++++ 8 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result create mode 100644 mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test diff --git a/mysql-test/include/mtr_warnings.sql b/mysql-test/include/mtr_warnings.sql index 768693bfecc..06a7b49e979 100644 --- a/mysql-test/include/mtr_warnings.sql +++ b/mysql-test/include/mtr_warnings.sql @@ -234,8 +234,6 @@ INSERT INTO global_suppressions VALUES ("WSREP: last inactive check more than .* skipping check"), ("WSREP: Gap in state sequence. Need state transfer."), ("WSREP: Failed to prepare for incremental state transfer: .*"), - ("WSREP: error executing 'SET GLOBAL innodb_disallow_writes=.*"), - ("WSREP: Failed to disallow InnoDB writes"), ("THE_LAST_SUPPRESSION")|| diff --git a/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result b/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result new file mode 100644 index 00000000000..bfb6b67b5d8 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result @@ -0,0 +1,45 @@ +# +# innodb_disallow_writes +# +# save the initial value +SET @innodb_disallow_writes_global_saved = @@global.innodb_disallow_writes; +# default +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +0 + +# scope +SELECT @@session.innodb_disallow_writes; +ERROR HY000: Variable 'innodb_disallow_writes' is a GLOBAL variable +SET @@global.innodb_disallow_writes=OFF; +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +0 +SET @@global.innodb_disallow_writes=ON; +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +1 + +# valid values +SET @@global.innodb_disallow_writes='OFF'; +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +0 +SET @@global.innodb_disallow_writes=ON; +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +1 +SET @@global.innodb_disallow_writes=default; +SELECT @@global.innodb_disallow_writes; +@@global.innodb_disallow_writes +0 + +# invalid values +SET @@global.innodb_disallow_writes=NULL; +ERROR 42000: Variable 'innodb_disallow_writes' can't be set to the value of 'NULL' +SET @@global.innodb_disallow_writes='junk'; +ERROR 42000: Variable 'innodb_disallow_writes' can't be set to the value of 'junk' + +# restore the initial value +SET @@global.innodb_disallow_writes = @innodb_disallow_writes_global_saved; +# End of test diff --git a/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test b/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test new file mode 100644 index 00000000000..b8e5c127377 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test @@ -0,0 +1,42 @@ +--source include/have_innodb_disallow_writes.inc + +--echo # +--echo # innodb_disallow_writes +--echo # + +--echo # save the initial value +SET @innodb_disallow_writes_global_saved = @@global.innodb_disallow_writes; + +--echo # default +SELECT @@global.innodb_disallow_writes; + +--echo +--echo # scope +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.innodb_disallow_writes; +SET @@global.innodb_disallow_writes=OFF; +SELECT @@global.innodb_disallow_writes; +SET @@global.innodb_disallow_writes=ON; +SELECT @@global.innodb_disallow_writes; + +--echo +--echo # valid values +SET @@global.innodb_disallow_writes='OFF'; +SELECT @@global.innodb_disallow_writes; +SET @@global.innodb_disallow_writes=ON; +SELECT @@global.innodb_disallow_writes; +SET @@global.innodb_disallow_writes=default; +SELECT @@global.innodb_disallow_writes; + +--echo +--echo # invalid values +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.innodb_disallow_writes=NULL; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.innodb_disallow_writes='junk'; + +--echo +--echo # restore the initial value +SET @@global.innodb_disallow_writes = @innodb_disallow_writes_global_saved; + +--echo # End of test diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d3f1980aeed..992b1e79b58 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -87,6 +87,12 @@ UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; /* In simulated aio, merge at most this many consecutive i/os */ #define OS_AIO_MERGE_N_CONSECUTIVE 64 +#ifdef WITH_INNODB_DISALLOW_WRITES +#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) +#else +#define WAIT_ALLOW_WRITES() do { } while (0) +#endif /* WITH_INNODB_DISALLOW_WRITES */ + /********************************************************************** InnoDB AIO Implementation: @@ -762,7 +768,9 @@ os_file_create_tmpfile(void) /*========================*/ { FILE* file = NULL; - int fd = innobase_mysql_tmpfile(); + int fd; + WAIT_ALLOW_WRITES(); + fd = innobase_mysql_tmpfile(); ut_ad(!srv_read_only_mode); @@ -1088,6 +1096,7 @@ os_file_create_directory( return(TRUE); #else int rcode; + WAIT_ALLOW_WRITES(); rcode = mkdir(pathname, 0770); @@ -1214,6 +1223,8 @@ os_file_create_simple_func( #else /* __WIN__ */ int create_flag; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1381,6 +1392,8 @@ os_file_create_simple_no_error_handling_func( int create_flag; ut_a(name); + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1672,6 +1685,8 @@ os_file_create_func( #else /* __WIN__ */ int create_flag; const char* mode_str = NULL; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; @@ -1851,6 +1866,7 @@ loop: goto loop; #else int ret; + WAIT_ALLOW_WRITES(); ret = unlink(name); @@ -1915,6 +1931,7 @@ loop: goto loop; #else int ret; + WAIT_ALLOW_WRITES(); ret = unlink(name); @@ -1968,6 +1985,7 @@ os_file_rename_func( return(FALSE); #else int ret; + WAIT_ALLOW_WRITES(); ret = rename(oldpath, newpath); @@ -2196,6 +2214,7 @@ os_file_set_eof( HANDLE h = (HANDLE) _get_osfhandle(fileno(file)); return(SetEndOfFile(h)); #else /* __WIN__ */ + WAIT_ALLOW_WRITES(); return(!ftruncate(fileno(file), ftell(file))); #endif /* __WIN__ */ } @@ -2290,6 +2309,7 @@ os_file_flush_func( return(FALSE); #else int ret; + WAIT_ALLOW_WRITES(); #if defined(HAVE_DARWIN_THREADS) # ifndef F_FULLFSYNC @@ -2971,6 +2991,7 @@ retry: return(FALSE); #else ssize_t ret; + WAIT_ALLOW_WRITES(); ret = os_file_pwrite(file, buf, n, offset); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 397616750ba..cdcadb90d64 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -2135,6 +2135,9 @@ int innobase_mysql_tmpfile(void) /*========================*/ { +#ifdef WITH_INNODB_DISALLOW_WRITES + os_event_wait(srv_allow_writes_event); +#endif /* WITH_INNODB_DISALLOW_WRITES */ int fd2 = -1; File fd; @@ -19194,6 +19197,40 @@ static MYSQL_SYSVAR_BOOL(disable_background_merge, NULL, NULL, FALSE); #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ +#ifdef WITH_INNODB_DISALLOW_WRITES +/******************************************************* + * innobase_disallow_writes variable definition * + *******************************************************/ + +/* Must always init to FALSE. */ +static my_bool innobase_disallow_writes = FALSE; + +/************************************************************************** +An "update" method for innobase_disallow_writes variable. */ +static +void +innobase_disallow_writes_update( +/*============================*/ + THD* thd, /* in: thread handle */ + st_mysql_sys_var* var, /* in: pointer to system + variable */ + void* var_ptr, /* out: pointer to dynamic + variable */ + const void* save) /* in: temporary storage */ +{ + *(my_bool*)var_ptr = *(my_bool*)save; + ut_a(srv_allow_writes_event); + if (*(my_bool*)var_ptr) + os_event_reset(srv_allow_writes_event); + else + os_event_set(srv_allow_writes_event); +} + +static MYSQL_SYSVAR_BOOL(disallow_writes, innobase_disallow_writes, + PLUGIN_VAR_NOCMDOPT, + "Tell InnoDB to stop any writes to disk", + NULL, innobase_disallow_writes_update, FALSE); +#endif /* WITH_INNODB_DISALLOW_WRITES */ static MYSQL_SYSVAR_BOOL(random_read_ahead, srv_random_read_ahead, PLUGIN_VAR_NOCMDARG, "Whether to use read ahead for random access within an extent.", @@ -19458,6 +19495,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(change_buffering_debug), MYSQL_SYSVAR(disable_background_merge), #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ +#ifdef WITH_INNODB_DISALLOW_WRITES + MYSQL_SYSVAR(disallow_writes), +#endif /* WITH_INNODB_DISALLOW_WRITES */ MYSQL_SYSVAR(random_read_ahead), MYSQL_SYSVAR(read_ahead_threshold), MYSQL_SYSVAR(read_only), diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 1303e481ae2..4c8a6f7b7b8 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -279,6 +279,10 @@ extern uint srv_flush_log_at_timeout; extern char srv_use_global_flush_log_at_trx_commit; extern char srv_adaptive_flushing; +#ifdef WITH_INNODB_DISALLOW_WRITES +/* When this event is reset we do not allow any file writes to take place. */ +extern os_event_t srv_allow_writes_event; +#endif /* WITH_INNODB_DISALLOW_WRITES */ /* If this flag is TRUE, then we will load the indexes' (and tables') metadata even if they are marked as "corrupted". Mostly it is for DBA to process corrupted index and table */ diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index db39a90b312..565c5720766 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -100,6 +100,12 @@ UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; /* In simulated aio, merge at most this many consecutive i/os */ #define OS_AIO_MERGE_N_CONSECUTIVE 64 +#ifdef WITH_INNODB_DISALLOW_WRITES +#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) +#else +#define WAIT_ALLOW_WRITES() do { } while (0) +#endif /* WITH_INNODB_DISALLOW_WRITES */ + /********************************************************************** InnoDB AIO Implementation: @@ -863,7 +869,9 @@ os_file_create_tmpfile(void) /*========================*/ { FILE* file = NULL; - int fd = innobase_mysql_tmpfile(); + int fd; + WAIT_ALLOW_WRITES(); + fd = innobase_mysql_tmpfile(); ut_ad(!srv_read_only_mode); @@ -1189,6 +1197,7 @@ os_file_create_directory( return(TRUE); #else int rcode; + WAIT_ALLOW_WRITES(); rcode = mkdir(pathname, 0770); @@ -1315,6 +1324,8 @@ os_file_create_simple_func( #else /* __WIN__ */ int create_flag; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1482,6 +1493,8 @@ os_file_create_simple_no_error_handling_func( int create_flag; ut_a(name); + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1828,6 +1841,8 @@ os_file_create_func( #else /* __WIN__ */ int create_flag; const char* mode_str = NULL; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; @@ -2017,6 +2032,7 @@ loop: goto loop; #else int ret; + WAIT_ALLOW_WRITES(); ret = unlink(name); @@ -2081,6 +2097,7 @@ loop: goto loop; #else int ret; + WAIT_ALLOW_WRITES(); ret = unlink(name); @@ -2134,6 +2151,7 @@ os_file_rename_func( return(FALSE); #else int ret; + WAIT_ALLOW_WRITES(); ret = rename(oldpath, newpath); @@ -2360,6 +2378,7 @@ os_file_set_eof( HANDLE h = (HANDLE) _get_osfhandle(fileno(file)); return(SetEndOfFile(h)); #else /* __WIN__ */ + WAIT_ALLOW_WRITES(); return(!ftruncate(fileno(file), ftell(file))); #endif /* __WIN__ */ } @@ -2379,6 +2398,7 @@ os_file_set_eof_at( return(SetFilePointerEx(file, li, &li2,FILE_BEGIN) && SetEndOfFile(file)); #else + WAIT_ALLOW_WRITES(); /* TODO: works only with -D_FILE_OFFSET_BITS=64 ? */ return(!ftruncate(file, new_len)); #endif @@ -2478,6 +2498,7 @@ os_file_flush_func( return(FALSE); #else int ret; + WAIT_ALLOW_WRITES(); #if defined(HAVE_DARWIN_THREADS) # ifndef F_FULLFSYNC @@ -3142,6 +3163,7 @@ retry: return(FALSE); #else ssize_t ret; + WAIT_ALLOW_WRITES(); ret = os_file_pwrite(file, buf, n, offset); diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 0ae2b49a52e..6665504e868 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -245,6 +245,10 @@ srv_printf_innodb_monitor() will request mutex acquisition with mutex_enter(), which will wait until it gets the mutex. */ #define MUTEX_NOWAIT(mutex_skipped) ((mutex_skipped) < MAX_MUTEX_NOWAIT) +#ifdef WITH_INNODB_DISALLOW_WRITES +UNIV_INTERN os_event_t srv_allow_writes_event; +#endif /* WITH_INNODB_DISALLOW_WRITES */ + /** The sort order table of the MySQL latin1_swedish_ci character set collation */ UNIV_INTERN const byte* srv_latin1_ordering; @@ -1132,6 +1136,14 @@ srv_init(void) dict_ind_init(); srv_conc_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + /* Writes have to be enabled on init or else we hang. Thus, we + always set the event here regardless of innobase_disallow_writes. + That flag will always be 0 at this point because it isn't settable + via my.cnf or command line arg. */ + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif /* WITH_INNODB_DISALLOW_WRITES */ /* Initialize some INFORMATION SCHEMA internal structures */ trx_i_s_cache_init(trx_i_s_cache); @@ -2130,7 +2142,20 @@ loop: if (sync_array_print_long_waits(&waiter, &sema) && sema == old_sema && os_thread_eq(waiter, old_waiter)) { +#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES) + if (srv_allow_writes_event->is_set) { +#endif /* WITH_WSREP */ fatal_cnt++; +#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES) + } else { + fprintf(stderr, + "WSREP: avoiding InnoDB self crash due to long " + "semaphore wait of > %lu seconds\n" + "Server is processing SST donor operation, " + "fatal_cnt now: %lu", + (ulong) srv_fatal_semaphore_wait_threshold, fatal_cnt); + } +#endif /* WITH_WSREP */ if (fatal_cnt > 10) { fprintf(stderr,