From 4ddb9deaf30bf3be664f3427f0b5393dc5a6c09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 30 Mar 2016 14:59:25 +0300 Subject: [PATCH 01/24] MDEV-9678: Data Directory bug MDEV-9833: Log files are opened using O_DIRECT causing problems if block size != 512 Fix typo. --- storage/xtradb/os/os0file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index b2b3e256211..2bb094e115d 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -1561,7 +1561,7 @@ os_file_set_nocache_if_needed(os_file_t file, const char* name, } if (srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT - || (type == OS_LOG_FILE + || (type == OS_DATA_FILE && (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT || (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)))) { os_file_set_nocache(file, name, mode_str); From 37a65e3335bd2dc28574b2f938d2a18b1b52f8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 30 Mar 2016 16:08:05 +0300 Subject: [PATCH 02/24] MDEV-9793: getting mysqld crypto key from key version failed Make sure that we read all possible encryption keys from checkpoint and if log block checksum does not match, print all found checkpoint encryption keys. --- storage/innobase/include/log0crypt.h | 10 +++++++++- storage/innobase/log/log0crypt.cc | 30 ++++++++++++++++++++++++++-- storage/innobase/log/log0recv.cc | 2 ++ storage/xtradb/include/log0crypt.h | 10 +++++++++- storage/xtradb/log/log0crypt.cc | 30 ++++++++++++++++++++++++++-- storage/xtradb/log/log0recv.cc | 2 ++ 6 files changed, 78 insertions(+), 6 deletions(-) diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 7e737853465..6b164e90d6e 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -117,4 +117,12 @@ log_crypt_print_error( /*==================*/ log_crypt_err_t err_info); /*!< out: error info */ +/*********************************************************************//** +Print checkpoint no from log block and all encryption keys from +checkpoints if they are present. Used for problem analysis. */ +void +log_crypt_print_checkpoint_keys( +/*============================*/ + const byte* log_block); + #endif // log0crypt.h diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 852148899e9..c404ad8dd0d 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -127,11 +127,34 @@ static const crypt_info_t* get_crypt_info( /*===========*/ - const byte* log_block) { + const byte* log_block) +{ ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); return get_crypt_info(checkpoint_no); } +/*********************************************************************//** +Print checkpoint no from log block and all encryption keys from +checkpoints if they are present. Used for problem analysis. */ +void +log_crypt_print_checkpoint_keys( +/*============================*/ + const byte* log_block) +{ + ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); + + if (crypt_info.size()) { + fprintf(stderr, "InnoDB: redo log checkpoint: %lu [ chk key ]: ", checkpoint_no); + for (size_t i = 0; i < crypt_info.size(); i++) { + struct crypt_info_t* it = &crypt_info[i]; + fprintf(stderr, "[ %lu %u ] ", + it->checkpoint_no, + it->key_version); + } + fprintf(stderr, "\n"); + } +} + /*********************************************************************//** Call AES CTR to encrypt/decrypt log blocks. */ static @@ -280,10 +303,13 @@ static bool add_crypt_info(crypt_info_t* info) { + const crypt_info_t* found=NULL; /* so that no one is searching array while we modify it */ ut_ad(mutex_own(&(log_sys->mutex))); - if (get_crypt_info(info->checkpoint_no) != NULL) { + found = get_crypt_info(info->checkpoint_no); + + if (found != NULL && found->checkpoint_no == info->checkpoint_no) { // already present... return true; } diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2304f4885c2..d574cd55397 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2714,6 +2714,8 @@ recv_scan_log_recs( /* Garbage or an incompletely written log block */ + /* Print checkpoint encryption keys if present */ + log_crypt_print_checkpoint_keys(log_block); finished = TRUE; if (maybe_encrypted) { diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 7e737853465..6b164e90d6e 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -117,4 +117,12 @@ log_crypt_print_error( /*==================*/ log_crypt_err_t err_info); /*!< out: error info */ +/*********************************************************************//** +Print checkpoint no from log block and all encryption keys from +checkpoints if they are present. Used for problem analysis. */ +void +log_crypt_print_checkpoint_keys( +/*============================*/ + const byte* log_block); + #endif // log0crypt.h diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index 852148899e9..c404ad8dd0d 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -127,11 +127,34 @@ static const crypt_info_t* get_crypt_info( /*===========*/ - const byte* log_block) { + const byte* log_block) +{ ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); return get_crypt_info(checkpoint_no); } +/*********************************************************************//** +Print checkpoint no from log block and all encryption keys from +checkpoints if they are present. Used for problem analysis. */ +void +log_crypt_print_checkpoint_keys( +/*============================*/ + const byte* log_block) +{ + ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); + + if (crypt_info.size()) { + fprintf(stderr, "InnoDB: redo log checkpoint: %lu [ chk key ]: ", checkpoint_no); + for (size_t i = 0; i < crypt_info.size(); i++) { + struct crypt_info_t* it = &crypt_info[i]; + fprintf(stderr, "[ %lu %u ] ", + it->checkpoint_no, + it->key_version); + } + fprintf(stderr, "\n"); + } +} + /*********************************************************************//** Call AES CTR to encrypt/decrypt log blocks. */ static @@ -280,10 +303,13 @@ static bool add_crypt_info(crypt_info_t* info) { + const crypt_info_t* found=NULL; /* so that no one is searching array while we modify it */ ut_ad(mutex_own(&(log_sys->mutex))); - if (get_crypt_info(info->checkpoint_no) != NULL) { + found = get_crypt_info(info->checkpoint_no); + + if (found != NULL && found->checkpoint_no == info->checkpoint_no) { // already present... return true; } diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index f98adbbca08..23ca8b1381f 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2786,6 +2786,8 @@ recv_scan_log_recs( /* Garbage or an incompletely written log block */ + /* Print checkpoint encryption keys if present */ + log_crypt_print_checkpoint_keys(log_block); finished = TRUE; if (maybe_encrypted) { From 2275640deb33af952ae094b2f742578639f1611b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 30 Mar 2016 14:23:37 +1100 Subject: [PATCH 03/24] Bug#20045167 UT_DELAY MISSING COMPILER BARRIER UT_RELAX_CPU(): Use a compiler barrier. ut_delay(): Remove the dummy global variable ut_always_false. RB: 11399 Reviewed-by: Jimmy Yang Backported from MySQL-5.7 - patch https://github.com/mysql/mysql-server/commit/5e3efb03962838a13afbf1579abbb96aee48ad64 Suggestion by Stewart Smith --- storage/innobase/include/ut0ut.h | 7 +------ storage/innobase/ut/ut0ut.cc | 7 ------- storage/xtradb/include/ut0ut.h | 7 +------ storage/xtradb/ut/ut0ut.cc | 7 ------- 4 files changed, 2 insertions(+), 26 deletions(-) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index e2635248b03..338cdf115d7 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -88,13 +88,8 @@ private: the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() -# elif defined(HAVE_ATOMIC_BUILTINS) -# define UT_RELAX_CPU() do { \ - volatile lint volatile_var; \ - os_compare_and_swap_lint(&volatile_var, 0, 1); \ - } while (0) # else -# define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ +# define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") # endif /*********************************************************************//** diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index a5970c1dc3f..a0fc154ce6b 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -45,9 +45,6 @@ Created 5/11/1994 Heikki Tuuri # include #endif /* UNIV_HOTBACKUP */ -/** A constant to prevent the compiler from optimizing ut_delay() away. */ -UNIV_INTERN ibool ut_always_false = FALSE; - #ifdef __WIN__ /*****************************************************************//** NOTE: The Windows epoch starts from 1601/01/01 whereas the Unix @@ -411,10 +408,6 @@ ut_delay( UT_RELAX_CPU(); } - if (ut_always_false) { - ut_always_false = (ibool) j; - } - return(j); } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index 9228c25d8be..fbebbc68628 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -80,18 +80,13 @@ private: # elif defined(HAVE_FAKE_PAUSE_INSTRUCTION) # define UT_RELAX_CPU() __asm__ __volatile__ ("rep; nop") -# elif defined(HAVE_ATOMIC_BUILTINS) -# define UT_RELAX_CPU() do { \ - volatile lint volatile_var; \ - os_compare_and_swap_lint(&volatile_var, 0, 1); \ - } while (0) # elif defined(HAVE_WINDOWS_ATOMICS) /* In the Win32 API, the x86 PAUSE instruction is executed by calling the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() # else -# define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ +# define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") # endif /*********************************************************************//** diff --git a/storage/xtradb/ut/ut0ut.cc b/storage/xtradb/ut/ut0ut.cc index 4eade1fe26e..5e85594ef44 100644 --- a/storage/xtradb/ut/ut0ut.cc +++ b/storage/xtradb/ut/ut0ut.cc @@ -46,9 +46,6 @@ Created 5/11/1994 Heikki Tuuri # include #endif /* UNIV_HOTBACKUP */ -/** A constant to prevent the compiler from optimizing ut_delay() away. */ -UNIV_INTERN ibool ut_always_false = FALSE; - #ifdef __WIN__ /*****************************************************************//** NOTE: The Windows epoch starts from 1601/01/01 whereas the Unix @@ -412,10 +409,6 @@ ut_delay( UT_RELAX_CPU(); } - if (ut_always_false) { - ut_always_false = (ibool) j; - } - return(j); } #endif /* !UNIV_HOTBACKUP */ From d4ba50477e6f0092a3b83528c02c0f30a2b708a3 Mon Sep 17 00:00:00 2001 From: Yasufumi Kinoshita Date: Wed, 30 Mar 2016 14:32:20 +1100 Subject: [PATCH 04/24] Some POWER specific optimizations Bug#18842925 : SET THREAD PRIORITY IN INNODB MUTEX SPINLOOP Like "pause" instruction for hyper-threading at Intel CPUs, POWER has special instructions only for hinting priority of hardware-threads. Approved by Sunny in rb#6256 Backport of the 5.7 fix - https://github.com/mysql/mysql-server/commit/c92102a6ef0f280bfb56e5585fca0d0cdcc34890 (excluded cache line size patch) Suggestion by Stewart Smith --- config.h.cmake | 1 + configure.cmake | 10 ++++++++++ storage/innobase/include/ut0ut.h | 8 ++++++++ storage/innobase/ut/ut0ut.cc | 4 ++++ storage/xtradb/include/ut0ut.h | 8 ++++++++ storage/xtradb/ut/ut0ut.cc | 4 ++++ 6 files changed, 35 insertions(+) diff --git a/config.h.cmake b/config.h.cmake index ce6f14723b8..493f90edc8c 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -225,6 +225,7 @@ #cmakedefine HAVE_PREAD 1 #cmakedefine HAVE_PAUSE_INSTRUCTION 1 #cmakedefine HAVE_FAKE_PAUSE_INSTRUCTION 1 +#cmakedefine HAVE_HMT_PRIORITY_INSTRUCTION 1 #cmakedefine HAVE_RDTSCLL 1 #cmakedefine HAVE_READ_REAL_TIME 1 #cmakedefine HAVE_PTHREAD_ATTR_CREATE 1 diff --git a/configure.cmake b/configure.cmake index a1fdb98389d..acb80965a64 100644 --- a/configure.cmake +++ b/configure.cmake @@ -855,6 +855,16 @@ IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) } " HAVE_FAKE_PAUSE_INSTRUCTION) ENDIF() + IF (NOT HAVE_PAUSE_INSTRUCTION) + CHECK_C_SOURCE_COMPILES(" + int main() + { + __asm__ __volatile__ (\"or 1,1,1\"); + __asm__ __volatile__ (\"or 2,2,2\"); + return 0; + } + " HAVE_HMT_PRIORITY_INSTRUCTION) + ENDIF() ENDIF() CHECK_SYMBOL_EXISTS(tcgetattr "termios.h" HAVE_TCGETATTR 1) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 338cdf115d7..03e4b5df833 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -92,6 +92,14 @@ private: # define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") # endif +# if defined(HAVE_HMT_PRIORITY_INSTRUCTION) +# define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") +# define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") +# else +# define UT_LOW_PRIORITY_CPU() ((void)0) +# define UT_RESUME_PRIORITY_CPU() ((void)0) +# endif + /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index a0fc154ce6b..85e77b6cced 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -401,6 +401,8 @@ ut_delay( { ulint i, j; + UT_LOW_PRIORITY_CPU(); + j = 0; for (i = 0; i < delay * 50; i++) { @@ -408,6 +410,8 @@ ut_delay( UT_RELAX_CPU(); } + UT_RESUME_PRIORITY_CPU(); + return(j); } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index fbebbc68628..6ea23514994 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -89,6 +89,14 @@ private: # define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") # endif +# if defined(HAVE_HMT_PRIORITY_INSTRUCTION) +# define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") +# define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") +# else +# define UT_LOW_PRIORITY_CPU() ((void)0) +# define UT_RESUME_PRIORITY_CPU() ((void)0) +# endif + /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. diff --git a/storage/xtradb/ut/ut0ut.cc b/storage/xtradb/ut/ut0ut.cc index 5e85594ef44..42fc04e5f81 100644 --- a/storage/xtradb/ut/ut0ut.cc +++ b/storage/xtradb/ut/ut0ut.cc @@ -402,6 +402,8 @@ ut_delay( { ulint i, j; + UT_LOW_PRIORITY_CPU(); + j = 0; for (i = 0; i < delay * 50; i++) { @@ -409,6 +411,8 @@ ut_delay( UT_RELAX_CPU(); } + UT_RESUME_PRIORITY_CPU(); + return(j); } #endif /* !UNIV_HOTBACKUP */ From 3d1a7cba71f6c843639f0b9a48b12017ff610112 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 30 Mar 2016 14:42:12 +1100 Subject: [PATCH 05/24] MDEV-8684: Remove delaying maths in ut_delay Also introduce compiler barrier properly on all architectures. --- storage/innobase/include/ut0ut.h | 6 ++++-- storage/innobase/ut/ut0ut.cc | 10 +++------- storage/xtradb/include/ut0ut.h | 6 ++++-- storage/xtradb/ut/ut0ut.cc | 10 +++------- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 03e4b5df833..1dcbae0d132 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -89,9 +89,11 @@ private: independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() # else -# define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") +# define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif +#define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") + # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) # define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") # define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") @@ -345,7 +347,7 @@ Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @return dummy value */ UNIV_INTERN -ulint +void ut_delay( /*=====*/ ulint delay); /*!< in: delay in microseconds on 100 MHz Pentium */ diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 85e77b6cced..bde40220db3 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -394,25 +394,21 @@ Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @return dummy value */ UNIV_INTERN -ulint +void ut_delay( /*=====*/ ulint delay) /*!< in: delay in microseconds on 100 MHz Pentium */ { - ulint i, j; + ulint i; UT_LOW_PRIORITY_CPU(); - j = 0; - for (i = 0; i < delay * 50; i++) { - j += i; UT_RELAX_CPU(); + UT_COMPILER_BARRIER(); } UT_RESUME_PRIORITY_CPU(); - - return(j); } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index 6ea23514994..a14014dc618 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -86,9 +86,11 @@ private: independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() # else -# define UT_RELAX_CPU() __asm__ __volatile__ ("":::"memory") +# define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif +#define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") + # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) # define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") # define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") @@ -337,7 +339,7 @@ Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @return dummy value */ UNIV_INTERN -ulint +void ut_delay( /*=====*/ ulint delay); /*!< in: delay in microseconds on 100 MHz Pentium */ diff --git a/storage/xtradb/ut/ut0ut.cc b/storage/xtradb/ut/ut0ut.cc index 42fc04e5f81..acedb56879a 100644 --- a/storage/xtradb/ut/ut0ut.cc +++ b/storage/xtradb/ut/ut0ut.cc @@ -395,25 +395,21 @@ Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @return dummy value */ UNIV_INTERN -ulint +void ut_delay( /*=====*/ ulint delay) /*!< in: delay in microseconds on 100 MHz Pentium */ { - ulint i, j; + ulint i; UT_LOW_PRIORITY_CPU(); - j = 0; - for (i = 0; i < delay * 50; i++) { - j += i; UT_RELAX_CPU(); + UT_COMPILER_BARRIER(); } UT_RESUME_PRIORITY_CPU(); - - return(j); } #endif /* !UNIV_HOTBACKUP */ From 64824a760d3ee4715d301dcdff541b66fac32992 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 30 Mar 2016 15:09:52 +1100 Subject: [PATCH 06/24] MDEV-8684: UT_RELAX_CPU on Power to non-empty expansion Using __ppc_get_timebase will translate to mfspr instruction The mfspr instruction will block FXU1 until complete but the other Pipelines are available for execution of instructions from other SMT threads on the same core. The latency time to read the timebase SPR is ~10 cycles. So any impact on other threads is limited other FXU1 only instructions (basically other mfspr/mtspr ops). Suggested by Steven J. Munroe, Linux on Power Toolchain Architect, Linux Technology Center IBM Corporation --- storage/innobase/include/ut0ut.h | 5 +++++ storage/xtradb/include/ut0ut.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 1dcbae0d132..9ed1a87508c 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -88,6 +88,11 @@ private: the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() +# elif defined(__powerpc__) +#include +# define UT_RELAX_CPU() do { \ + volatile lint volatile_var = __ppc_get_timebase(); \ + } while (0) # else # define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index a14014dc618..328ccdf7430 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -85,6 +85,11 @@ private: the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ # define UT_RELAX_CPU() YieldProcessor() +# elif defined(__powerpc__) +#include +# define UT_RELAX_CPU() do { \ + volatile lint volatile_var = __ppc_get_timebase(); \ + } while (0) # else # define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif From c395aad668bdf675b1a35f093c9f1d8d8570b7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 31 Mar 2016 13:12:48 +0300 Subject: [PATCH 07/24] MDEV-9840: Test encryption.innodb-log-encrypt-crash fails on buildbot Problem: We created more than 5 encryption keys for redo-logs. Idea was that we do not anymore create more than one encryption key for redo-logs but if existing checkpoint from earlier MariaDB contains more keys, we should read all of them. Fix: Add new encryption key to memory structure only if there currently has none or if we are reading checkpoint from the log. Checkpoint from older MariaDB version could contain more than one key. --- storage/innobase/log/log0crypt.cc | 15 +++++++++++---- storage/xtradb/log/log0crypt.cc | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index c404ad8dd0d..db2e84d7e45 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -301,7 +301,10 @@ Add crypt info to set if it is not already present @return true if successfull, false if not- */ static bool -add_crypt_info(crypt_info_t* info) +add_crypt_info( +/*===========*/ + crypt_info_t* info, /*!< in: crypt info */ + bool checkpoint_read)/*!< in: do we read checkpoint */ { const crypt_info_t* found=NULL; /* so that no one is searching array while we modify it */ @@ -309,7 +312,11 @@ add_crypt_info(crypt_info_t* info) found = get_crypt_info(info->checkpoint_no); - if (found != NULL && found->checkpoint_no == info->checkpoint_no) { + /* If one crypt info is found then we add a new one only if we + are reading checkpoint from the log. New checkpoints will always + use the first created crypt info. */ + if (found != NULL && + ( found->checkpoint_no == info->checkpoint_no || !checkpoint_read)) { // already present... return true; } @@ -382,7 +389,7 @@ log_crypt_set_ver_and_key( } - add_crypt_info(&info); + add_crypt_info(&info, false); } /******************************************************** @@ -540,7 +547,7 @@ log_crypt_read_checkpoint_buf( memcpy(info.crypt_msg, buf + 8, MY_AES_BLOCK_SIZE); memcpy(info.crypt_nonce, buf + 24, MY_AES_BLOCK_SIZE); - if (!add_crypt_info(&info)) { + if (!add_crypt_info(&info, true)) { return false; } buf += LOG_CRYPT_ENTRY_SIZE; diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index c404ad8dd0d..db2e84d7e45 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -301,7 +301,10 @@ Add crypt info to set if it is not already present @return true if successfull, false if not- */ static bool -add_crypt_info(crypt_info_t* info) +add_crypt_info( +/*===========*/ + crypt_info_t* info, /*!< in: crypt info */ + bool checkpoint_read)/*!< in: do we read checkpoint */ { const crypt_info_t* found=NULL; /* so that no one is searching array while we modify it */ @@ -309,7 +312,11 @@ add_crypt_info(crypt_info_t* info) found = get_crypt_info(info->checkpoint_no); - if (found != NULL && found->checkpoint_no == info->checkpoint_no) { + /* If one crypt info is found then we add a new one only if we + are reading checkpoint from the log. New checkpoints will always + use the first created crypt info. */ + if (found != NULL && + ( found->checkpoint_no == info->checkpoint_no || !checkpoint_read)) { // already present... return true; } @@ -382,7 +389,7 @@ log_crypt_set_ver_and_key( } - add_crypt_info(&info); + add_crypt_info(&info, false); } /******************************************************** @@ -540,7 +547,7 @@ log_crypt_read_checkpoint_buf( memcpy(info.crypt_msg, buf + 8, MY_AES_BLOCK_SIZE); memcpy(info.crypt_nonce, buf + 24, MY_AES_BLOCK_SIZE); - if (!add_crypt_info(&info)) { + if (!add_crypt_info(&info, true)) { return false; } buf += LOG_CRYPT_ENTRY_SIZE; From 26c38de804ecb87eae7fbe6ac32dadebb7803b4d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 1 Apr 2016 09:03:57 +1100 Subject: [PATCH 08/24] MDEV-8684: Use POWER wrappers rather than direct asm i.e. __ppc_set_ppr_low rather than 'or 1,1,1' --- configure.cmake | 5 +++-- storage/innobase/include/ut0ut.h | 5 +++-- storage/xtradb/include/ut0ut.h | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/configure.cmake b/configure.cmake index acb80965a64..5b8bc3688c7 100644 --- a/configure.cmake +++ b/configure.cmake @@ -857,10 +857,11 @@ IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) ENDIF() IF (NOT HAVE_PAUSE_INSTRUCTION) CHECK_C_SOURCE_COMPILES(" + #include int main() { - __asm__ __volatile__ (\"or 1,1,1\"); - __asm__ __volatile__ (\"or 2,2,2\"); + __ppc_set_ppr_low(); + __ppc_set_ppr_med(); return 0; } " HAVE_HMT_PRIORITY_INSTRUCTION) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 9ed1a87508c..1334a6000be 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -100,8 +100,9 @@ private: #define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -# define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") -# define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") +#include +# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() +# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() # else # define UT_LOW_PRIORITY_CPU() ((void)0) # define UT_RESUME_PRIORITY_CPU() ((void)0) diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index 328ccdf7430..722acdb607e 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -97,8 +97,9 @@ private: #define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -# define UT_LOW_PRIORITY_CPU() __asm__ __volatile__ ("or 1,1,1") -# define UT_RESUME_PRIORITY_CPU() __asm__ __volatile__ ("or 2,2,2") +#include +# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() +# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() # else # define UT_LOW_PRIORITY_CPU() ((void)0) # define UT_RESUME_PRIORITY_CPU() ((void)0) From 04737330bea89437d8069d8eeb4bc484213dfdf0 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 3 Apr 2016 20:19:59 +0300 Subject: [PATCH 09/24] MDEV-9860: TokuDB ORDER BY DESC query is slower in 10.1 with ICP ON Implement ha_tokudb::cancel_pushed_idx_cond(). This is a conservative fix which follows the approach from the previous patch for: BUG#1000051: Query with simple join and ORDER BY takes thousands times... --- storage/tokudb/ha_tokudb.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/tokudb/ha_tokudb.h b/storage/tokudb/ha_tokudb.h index e263cabb0d1..5b387924b1e 100644 --- a/storage/tokudb/ha_tokudb.h +++ b/storage/tokudb/ha_tokudb.h @@ -602,6 +602,12 @@ public: // ICP introduced in MariaDB 5.5 Item* idx_cond_push(uint keyno, class Item* idx_cond); +#ifdef MARIADB_BASE_VERSION + void cancel_pushed_idx_cond() + { + invalidate_icp(); + } +#endif #if TOKU_INCLUDE_ALTER_56 public: From fd7a8d18ea3a844be1a82490be3c154549dfee47 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 6 Apr 2016 13:38:22 +0200 Subject: [PATCH 10/24] Fix compile error in UT_COMPILER_BARRIER on Visual Studio compiler. --- storage/innobase/include/ut0ut.h | 8 +++++++- storage/xtradb/include/ut0ut.h | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 1334a6000be..5c4a358f9a6 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -97,7 +97,13 @@ private: # define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif -#define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +#if defined (__GNUC__) +# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +#elif defined (_MSC_VER) +# define UT_COMPILER_BARRIER() MemoryBarrier() +#else +# define UT_COMPILER_BARRIER() +#endif # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) #include diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index 722acdb607e..053eb78e42b 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -94,7 +94,13 @@ private: # define UT_RELAX_CPU() ((void)0) /* avoid warning for an empty statement */ # endif -#define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +#if defined (__GNUC__) +# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +#elif defined (_MSC_VER) +# define UT_COMPILER_BARRIER() MemoryBarrier() +#else +# define UT_COMPILER_BARRIER() +#endif # if defined(HAVE_HMT_PRIORITY_INSTRUCTION) #include From 4b6a3518e4dc9088d1f42cd9bc487d06137d2760 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 6 Apr 2016 14:15:44 +0200 Subject: [PATCH 11/24] Use _ReadWriteBarrier() rather than MemoryBarrier() for preventing compile optimization --- storage/innobase/include/ut0ut.h | 2 +- storage/xtradb/include/ut0ut.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 5c4a358f9a6..176f132704a 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -100,7 +100,7 @@ private: #if defined (__GNUC__) # define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") #elif defined (_MSC_VER) -# define UT_COMPILER_BARRIER() MemoryBarrier() +# define UT_COMPILER_BARRIER() _ReadWriteBarrier() #else # define UT_COMPILER_BARRIER() #endif diff --git a/storage/xtradb/include/ut0ut.h b/storage/xtradb/include/ut0ut.h index 053eb78e42b..980ac337002 100644 --- a/storage/xtradb/include/ut0ut.h +++ b/storage/xtradb/include/ut0ut.h @@ -97,7 +97,7 @@ private: #if defined (__GNUC__) # define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") #elif defined (_MSC_VER) -# define UT_COMPILER_BARRIER() MemoryBarrier() +# define UT_COMPILER_BARRIER() _ReadWriteBarrier() #else # define UT_COMPILER_BARRIER() #endif From 293cb04959172f65b9d62124fd20339c47658eb0 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 7 Apr 2016 19:26:25 +0300 Subject: [PATCH 12/24] MDEV-9621 INSERT DELAYED fails on insert for tables with many columns This fix also fixes a connection hang when trying to do INSERT DELAYED to a crashed table. Added crash_mysqld.inc to allow easy crash+restart of mysqld --- mysql-test/include/crash_mysqld.inc | 18 +++++++++++++ mysql-test/r/delayed.result | 34 ++++++++++++++++++++++++ mysql-test/t/delayed.test | 41 +++++++++++++++++++++++++++++ sql/sql_base.cc | 9 +++++-- sql/sql_base.h | 4 +++ sql/sql_insert.cc | 19 ++++++++----- sql/sql_parse.cc | 1 + sql/table.h | 1 + 8 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 mysql-test/include/crash_mysqld.inc diff --git a/mysql-test/include/crash_mysqld.inc b/mysql-test/include/crash_mysqld.inc new file mode 100644 index 00000000000..4190d24d801 --- /dev/null +++ b/mysql-test/include/crash_mysqld.inc @@ -0,0 +1,18 @@ +# Crash mysqld hard and wait until it's restarted + +--source include/have_debug_sync.inc +--source include/not_embedded.inc + +# Write file to make mysql-test-run.pl expect crash and restart +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +# Setup the mysqld to crash at shutdown +SET debug_dbug="d,crash_shutdown"; +--error 2013 +shutdown; + +# Turn on reconnect +--enable_reconnect + +# Call script that will poll the server waiting for it to be back online again +--source include/wait_until_connected_again.inc diff --git a/mysql-test/r/delayed.result b/mysql-test/r/delayed.result index 98c8b599e88..8cc5645eeda 100644 --- a/mysql-test/r/delayed.result +++ b/mysql-test/r/delayed.result @@ -466,3 +466,37 @@ commit; # Reaping ALTER TABLE: # Connection 'default'. drop tables tm, t1, t2; +# +# MDEV-9621 INSERT DELAYED fails on insert for tables with many columns +# +CREATE TABLE t1 ( +a int,b int,c int,d int,e int,f int,g int,h int,i int,j int,k int,l int,m int,n int,o int,p int,q int,r int,s int,t int,u int,v int,x int,y int,z int +) ENGINE=MyISAM; +INSERT DELAYED INTO t1 (a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,x,y,z) +values (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); +INSERT DELAYED INTO t1 (a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,x,y,z) +values (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); +drop table t1; +# +# INSERT DELAYED hangs if table was crashed +# +create table t1 (a int, b int) engine=myisam; +insert into t1 values (1,1); +SET debug_dbug="d,crash_shutdown"; +shutdown; +ERROR HY000: Lost connection to MySQL server during query +call mtr.add_suppression(" marked as crashed and should be repaired"); +call mtr.add_suppression("Checking table"); +insert delayed into t1 values (2,2); +Warnings: +Error 145 Table './test/t1' is marked as crashed and should be repaired +Error 1194 Table 't1' is marked as crashed and should be repaired +Error 1034 1 client is using or hasn't closed the table properly +insert delayed into t1 values (3,3); +flush tables t1; +select * from t1; +a b +1 1 +2 2 +3 3 +drop table t1; diff --git a/mysql-test/t/delayed.test b/mysql-test/t/delayed.test index 2886dff8f91..644cbfbe6ce 100644 --- a/mysql-test/t/delayed.test +++ b/mysql-test/t/delayed.test @@ -1,5 +1,12 @@ # delayed works differently in embedded server --source include/not_embedded.inc +# Don't test this under valgrind, memory leaks will occur +--source include/not_valgrind.inc +# Avoid CrashReporter popup on Mac +--source include/not_crashrep.inc +# Binary must be compiled with debug for crash to occur +--source include/have_debug_sync.inc + # # test of DELAYED insert and timestamps # (Can't be tested with purify :( ) @@ -629,3 +636,37 @@ disconnect con1; --echo # Connection 'default'. connection default; drop tables tm, t1, t2; + +--echo # +--echo # MDEV-9621 INSERT DELAYED fails on insert for tables with many columns +--echo # + +CREATE TABLE t1 ( + a int,b int,c int,d int,e int,f int,g int,h int,i int,j int,k int,l int,m int,n int,o int,p int,q int,r int,s int,t int,u int,v int,x int,y int,z int +) ENGINE=MyISAM; + +INSERT DELAYED INTO t1 (a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,x,y,z) +values (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); +INSERT DELAYED INTO t1 (a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,x,y,z) +values (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); +drop table t1; + +--echo # +--echo # INSERT DELAYED hangs if table was crashed +--echo # + +create table t1 (a int, b int) engine=myisam; +insert into t1 values (1,1); + +# Will come back with t1 crashed. +--source include/crash_mysqld.inc + +call mtr.add_suppression(" marked as crashed and should be repaired"); +call mtr.add_suppression("Checking table"); + +--replace_result '\\' '/' +insert delayed into t1 values (2,2); +insert delayed into t1 values (3,3); +flush tables t1; +select * from t1; +drop table t1; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 98c63a92a86..e808fbafb50 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -2570,8 +2570,13 @@ retry_share: (void) ot_ctx->request_backoff_action(Open_table_context::OT_DISCOVER, table_list); else if (share->crashed) - (void) ot_ctx->request_backoff_action(Open_table_context::OT_REPAIR, - table_list); + { + if (!(flags & MYSQL_OPEN_IGNORE_REPAIR)) + (void) ot_ctx->request_backoff_action(Open_table_context::OT_REPAIR, + table_list); + else + table_list->crashed= 1; /* Mark that table was crashed */ + } goto err_lock; } if (open_table_entry_fini(thd, share, table)) diff --git a/sql/sql_base.h b/sql/sql_base.h index 7407e230419..9e37a43aab8 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -107,6 +107,10 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, table flush, wait on thr_lock.c locks) while opening and locking table. */ #define MYSQL_OPEN_IGNORE_KILLED 0x8000 +/** + Don't try to auto-repair table +*/ +#define MYSQL_OPEN_IGNORE_REPAIR 0x10000 /** Please refer to the internals manual. */ #define MYSQL_OPEN_REOPEN (MYSQL_OPEN_IGNORE_FLUSH |\ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index f8a755c8d5e..9b4238b513d 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -2014,6 +2014,7 @@ public: mysql_cond_t cond, cond_client; volatile uint tables_in_use,stacked_inserts; volatile bool status; + bool retry; /** When the handler thread starts, it clones a metadata lock ticket which protects against GRL and ticket for the table to be inserted. @@ -2038,7 +2039,7 @@ public: Delayed_insert(SELECT_LEX *current_select) :locks_in_memory(0), table(0),tables_in_use(0),stacked_inserts(0), - status(0), handler_thread_initialized(FALSE), group_count(0) + status(0), retry(0), handler_thread_initialized(FALSE), group_count(0) { DBUG_ENTER("Delayed_insert constructor"); thd.security_ctx->user=(char*) delayed_user; @@ -2291,7 +2292,7 @@ bool delayed_get_table(THD *thd, MDL_request *grl_protection_request, } if (di->thd.killed) { - if (di->thd.is_error()) + if (di->thd.is_error() && ! di->retry) { /* Copy the error message. Note that we don't treat fatal @@ -2517,7 +2518,7 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) copy->vcol_set= copy->def_vcol_set; } copy->tmp_set.bitmap= 0; // To catch errors - bzero((char*) bitmap, share->column_bitmap_size + (share->vfields ? 3 : 2)); + bzero((char*) bitmap, share->column_bitmap_size * (share->vfields ? 3 : 2)); copy->read_set= ©->def_read_set; copy->write_set= ©->def_write_set; @@ -2526,7 +2527,6 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) /* Got fatal error */ error: tables_in_use--; - status=1; mysql_cond_signal(&cond); // Inform thread about abort DBUG_RETURN(0); } @@ -2768,13 +2768,20 @@ bool Delayed_insert::open_and_lock_table() /* Use special prelocking strategy to get ER_DELAYED_NOT_SUPPORTED error for tables with engines which don't support delayed inserts. + + We can't do auto-repair in insert delayed thread, as it would hang + when trying to an exclusive MDL_LOCK on the table during repair + as the connection thread has a SHARED_WRITE lock. */ if (!(table= open_n_lock_single_table(&thd, &table_list, TL_WRITE_DELAYED, - MYSQL_OPEN_IGNORE_GLOBAL_READ_LOCK, + MYSQL_OPEN_IGNORE_GLOBAL_READ_LOCK | + MYSQL_OPEN_IGNORE_REPAIR, &prelocking_strategy))) { - thd.fatal_error(); // Abort waiting inserts + /* If table was crashed, then upper level should retry open+repair */ + retry= table_list.crashed; + thd.fatal_error(); // Abort waiting inserts return TRUE; } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 6fcb549f097..4c8bb05cc39 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4925,6 +4925,7 @@ end_with_restore_list: } case SQLCOM_SHUTDOWN: #ifndef EMBEDDED_LIBRARY + DBUG_EXECUTE_IF("crash_shutdown", DBUG_SUICIDE();); if (check_global_access(thd,SHUTDOWN_ACL)) goto error; kill_mysql(); diff --git a/sql/table.h b/sql/table.h index 023d5e542b8..eb4076e02e2 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2005,6 +2005,7 @@ struct TABLE_LIST bool updating; /* for replicate-do/ignore table */ bool force_index; /* prefer index over table scan */ bool ignore_leaves; /* preload only non-leaf nodes */ + bool crashed; /* Table was found crashed */ table_map dep_tables; /* tables the table depends on */ table_map on_expr_dep_tables; /* tables on expression depends on */ struct st_nested_join *nested_join; /* if the element is a nested join */ From 3f6125129f986b2d9bd4a002c30e35e6963b0e3f Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 7 Apr 2016 20:38:21 +0300 Subject: [PATCH 13/24] Fixed results --- storage/tokudb/mysql-test/tokudb_bugs/r/simple_icp.result | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/simple_icp.result b/storage/tokudb/mysql-test/tokudb_bugs/r/simple_icp.result index 2975d7d3116..12fec571d87 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/r/simple_icp.result +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/simple_icp.result @@ -110,7 +110,7 @@ a b c d e 5 1 10 NULL NULL show status like '%Handler_read_prev%'; Variable_name Value -Handler_read_prev 41 +Handler_read_prev 800 flush status; show status like '%Handler_read_prev%'; Variable_name Value @@ -142,7 +142,7 @@ a b c d e 20 1 10 NULL NULL show status like '%Handler_read_prev%'; Variable_name Value -Handler_read_prev 21 +Handler_read_prev 400 flush status; show status like '%Handler_read_next%'; Variable_name Value From 1cf852d874b0e82ebfa3854300abaacd04d3eb01 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 7 Apr 2016 14:44:29 +0200 Subject: [PATCH 14/24] MDEV-9383: Server fails to read master.info after upgrade 10.0 -> 10.1 In some cases, MariaDB 10.0 could write a master.info file that was read incorrectly by 10.1 and could cause server to fail to start after an upgrade. (If writing a new master.info file that is shorter than the old, extra junk may remain at the end of the file. This is handled properly in 10.1 with an END_MARKER line, but this line is not written by 10.0. The fix here is to make 10.1 robust at reading the master.info files written by 10.0). Fix several things around reading master.info and read_mi_key_from_file(): - read_mi_key_from_file() did not distinguish between a line with and without an eqals '=' sign. - If a line was empty, read_mi_key_from_file() would incorrectly return the key from the previous call. - An extra using_gtid=X line left-over by MariaDB 10.0 might incorrectly be read and overwrite the correct value. - Fix incorrect usage of strncmp() which should be strcmp(). - Add test cases. --- mysql-test/std_data/bad2_master.info | 35 ++++ mysql-test/std_data/bad3_master.info | 37 ++++ mysql-test/std_data/bad4_master.info | 35 ++++ mysql-test/std_data/bad5_master.info | 35 ++++ mysql-test/std_data/bad6_master.info | 36 ++++ mysql-test/std_data/bad_master.info | 35 ++++ .../rpl/r/rpl_upgrade_master_info.result | 88 ++++++++++ .../suite/rpl/t/rpl_upgrade_master_info.test | 163 ++++++++++++++++++ sql/rpl_mi.cc | 87 ++++++---- 9 files changed, 515 insertions(+), 36 deletions(-) create mode 100644 mysql-test/std_data/bad2_master.info create mode 100644 mysql-test/std_data/bad3_master.info create mode 100644 mysql-test/std_data/bad4_master.info create mode 100644 mysql-test/std_data/bad5_master.info create mode 100644 mysql-test/std_data/bad6_master.info create mode 100644 mysql-test/std_data/bad_master.info create mode 100644 mysql-test/suite/rpl/r/rpl_upgrade_master_info.result create mode 100644 mysql-test/suite/rpl/t/rpl_upgrade_master_info.test diff --git a/mysql-test/std_data/bad2_master.info b/mysql-test/std_data/bad2_master.info new file mode 100644 index 00000000000..61722562748 --- /dev/null +++ b/mysql-test/std_data/bad2_master.info @@ -0,0 +1,35 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 +=0 diff --git a/mysql-test/std_data/bad3_master.info b/mysql-test/std_data/bad3_master.info new file mode 100644 index 00000000000..6e632cd9a49 --- /dev/null +++ b/mysql-test/std_data/bad3_master.info @@ -0,0 +1,37 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 + + +0 diff --git a/mysql-test/std_data/bad4_master.info b/mysql-test/std_data/bad4_master.info new file mode 100644 index 00000000000..87572efc8a4 --- /dev/null +++ b/mysql-test/std_data/bad4_master.info @@ -0,0 +1,35 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 +d=1 diff --git a/mysql-test/std_data/bad5_master.info b/mysql-test/std_data/bad5_master.info new file mode 100644 index 00000000000..4ea8113250b --- /dev/null +++ b/mysql-test/std_data/bad5_master.info @@ -0,0 +1,35 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 +using_gtid diff --git a/mysql-test/std_data/bad6_master.info b/mysql-test/std_data/bad6_master.info new file mode 100644 index 00000000000..0f48f4871f0 --- /dev/null +++ b/mysql-test/std_data/bad6_master.info @@ -0,0 +1,36 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 +END_MARKER +do_domain_ids=20 Hulubulu!!?! diff --git a/mysql-test/std_data/bad_master.info b/mysql-test/std_data/bad_master.info new file mode 100644 index 00000000000..1541fdf2c61 --- /dev/null +++ b/mysql-test/std_data/bad_master.info @@ -0,0 +1,35 @@ +33 +mysql-bin.000001 +4 +127.0.0.1 +root + +3310 +60 +0 + + + + + +0 +1800.000 + +0 + +0 + + + + + + + + + + + + + +using_gtid=1 + diff --git a/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result b/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result new file mode 100644 index 00000000000..f966f18964b --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result @@ -0,0 +1,88 @@ +include/master-slave.inc +[connection master] +*** MDEV-9383: Server fails to read master.info after upgrade 10.0 -> 10.1 *** +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=CURRENT_POS; +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +CREATE TABLE t1 (a INT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1; +a +1 +include/stop_slave.inc +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +INSERT INTO t1 VALUES (2); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +include/stop_slave.inc +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +INSERT INTO t1 VALUES (3); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +include/stop_slave.inc +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +INSERT INTO t1 VALUES (4); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +include/stop_slave.inc +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +INSERT INTO t1 VALUES (5); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +include/stop_slave.inc +include/rpl_stop_server.inc [server_number=2] +include/rpl_start_server.inc [server_number=2] +INSERT INTO t1 VALUES (6); +include/save_master_gtid.inc +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test b/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test new file mode 100644 index 00000000000..e81e7c0d714 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test @@ -0,0 +1,163 @@ +--source include/master-slave.inc + +--echo *** MDEV-9383: Server fails to read master.info after upgrade 10.0 -> 10.1 *** + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=CURRENT_POS; +--let $datadir= `SELECT @@datadir` + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +CREATE TABLE t1 (a INT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1; + +--source include/stop_slave.inc + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad2_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +INSERT INTO t1 VALUES (2); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + +--source include/stop_slave.inc + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad3_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +INSERT INTO t1 VALUES (3); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + +--source include/stop_slave.inc + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad4_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +INSERT INTO t1 VALUES (4); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + +--source include/stop_slave.inc + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad5_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +INSERT INTO t1 VALUES (5); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + +--source include/stop_slave.inc + +--let $rpl_server_number= 2 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master.info +--copy_file $MYSQL_TEST_DIR/std_data/bad6_master.info $datadir/master.info + +--let $rpl_server_number= 2 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +--connection master +INSERT INTO t1 VALUES (6); +--source include/save_master_gtid.inc + +--connection slave +# Fix the port after we replaced master.info. +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + + +# Cleanup +--connection master +DROP TABLE t1; +--source include/rpl_end.inc diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index df721342d1d..02dbac46eb5 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -205,43 +205,56 @@ void init_master_log_pos(Master_info* mi) /** Parses the IO_CACHE for "key=" and returns the "key". + If no '=' found, returns the whole line (for END_MARKER). @param key [OUT] Key buffer @param max_size [IN] Maximum buffer size @param f [IN] IO_CACHE file + @param found_equal [OUT] Set true if a '=' was found. @retval 0 Either "key=" or '\n' found @retval 1 EOF */ -static int read_mi_key_from_file(char *key, int max_size, IO_CACHE *f) +static int +read_mi_key_from_file(char *key, int max_size, IO_CACHE *f, bool *found_equal) { int i= 0, c; - char *last_p; DBUG_ENTER("read_key_from_file"); - while (((c= my_b_get(f)) != '\n') && (c != my_b_EOF)) - { - last_p= key + i; - - if (i < max_size) - { - if (c == '=') - { - /* We found '=', replace it by 0 and return. */ - *last_p= 0; - DBUG_RETURN(0); - } - else - *last_p= c; - } - ++i; - } - - if (c == my_b_EOF) + *found_equal= false; + if (max_size <= 0) DBUG_RETURN(1); - - DBUG_RETURN(0); + for (;;) + { + if (i >= max_size-1) + { + key[i] = '\0'; + DBUG_RETURN(0); + } + c= my_b_get(f); + if (c == my_b_EOF) + { + DBUG_RETURN(1); + } + else if (c == '\n') + { + key[i]= '\0'; + DBUG_RETURN(0); + } + else if (c == '=') + { + key[i]= '\0'; + *found_equal= true; + DBUG_RETURN(0); + } + else + { + key[i]= c; + ++i; + } + } + /* NotReached */ } enum { @@ -539,6 +552,10 @@ file '%s')", fname); if (lines >= LINE_FOR_LAST_MYSQL_FUTURE) { uint i; + bool got_eq; + bool seen_using_gtid= false; + bool seen_do_domain_ids=false, seen_ignore_domain_ids=false; + /* Skip lines used by / reserved for MySQL >= 5.6. */ for (i= LINE_FOR_FIRST_MYSQL_5_6; i <= LINE_FOR_LAST_MYSQL_FUTURE; ++i) { @@ -551,11 +568,12 @@ file '%s')", fname); for "key=" and returns the "key" if found. The "value" can then the parsed on case by case basis. The "unknown" lines would be ignored to facilitate downgrades. + 10.0 does not have the END_MARKER before any left-overs at the end + of the file. So ignore any but the first occurrence of a key. */ - while (!read_mi_key_from_file(buf, sizeof(buf), &mi->file)) + while (!read_mi_key_from_file(buf, sizeof(buf), &mi->file, &got_eq)) { - /* using_gtid */ - if (!strncmp(buf, STRING_WITH_LEN("using_gtid"))) + if (got_eq && !seen_using_gtid && !strcmp(buf, "using_gtid")) { int val; if (!init_intvar_from_file(&val, &mi->file, 0)) @@ -566,15 +584,13 @@ file '%s')", fname); mi->using_gtid= Master_info::USE_GTID_SLAVE_POS; else mi->using_gtid= Master_info::USE_GTID_NO; - continue; + seen_using_gtid= true; } else { sql_print_error("Failed to initialize master info using_gtid"); goto errwithmsg; } } - - /* DO_DOMAIN_IDS */ - if (!strncmp(buf, STRING_WITH_LEN("do_domain_ids"))) + else if (got_eq && !seen_do_domain_ids && !strcmp(buf, "do_domain_ids")) { if (mi->domain_id_filter.init_ids(&mi->file, Domain_id_filter::DO_DOMAIN_IDS)) @@ -582,11 +598,10 @@ file '%s')", fname); sql_print_error("Failed to initialize master info do_domain_ids"); goto errwithmsg; } - continue; + seen_do_domain_ids= true; } - - /* IGNORE_DOMAIN_IDS */ - if (!strncmp(buf, STRING_WITH_LEN("ignore_domain_ids"))) + else if (got_eq && !seen_ignore_domain_ids && + !strcmp(buf, "ignore_domain_ids")) { if (mi->domain_id_filter.init_ids(&mi->file, Domain_id_filter::IGNORE_DOMAIN_IDS)) @@ -595,9 +610,9 @@ file '%s')", fname); "ignore_domain_ids"); goto errwithmsg; } - continue; + seen_ignore_domain_ids= true; } - else if (!strncmp(buf, STRING_WITH_LEN("END_MARKER"))) + else if (!got_eq && !strcmp(buf, "END_MARKER")) { /* Guard agaist extra left-overs at the end of file, in case a later From 47e07178060cf78d1914f83ab1daff4f557f8615 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Wed, 27 Apr 2016 11:08:46 +0400 Subject: [PATCH 15/24] MDEV-9792 Backport MDEV-8713 to 10.1. --- client/mysqlbinlog.cc | 521 ++++++++++++++++------- mysql-test/r/mysqlbinlog_raw_mode.result | 274 ++++++++++++ mysql-test/t/mysqlbinlog_raw_mode.test | 387 +++++++++++++++++ sql/sql_priv.h | 3 +- 4 files changed, 1031 insertions(+), 154 deletions(-) create mode 100644 mysql-test/r/mysqlbinlog_raw_mode.result create mode 100644 mysql-test/t/mysqlbinlog_raw_mode.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index bb0dd4040af..c6a7c622ecd 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -73,6 +73,8 @@ ulong opt_binlog_rows_event_max_size; uint test_flags = 0; static uint opt_protocol= 0; static FILE *result_file; +static char *result_file_name= 0; +static const char *output_prefix= ""; #ifndef DBUG_OFF static const char* default_dbug_option = "d:t:o,/tmp/mysqlbinlog.trace"; @@ -96,6 +98,8 @@ static char* database= 0; static my_bool force_opt= 0, short_form= 0, remote_opt= 0; static my_bool debug_info_flag, debug_check_flag; static my_bool force_if_open_opt= 1; +static my_bool opt_raw_mode= 0, opt_stop_never= 0; +static ulong opt_stop_never_slave_server_id= 0; static my_bool opt_verify_binlog_checksum= 1; static ulonglong offset = 0; static char* host = 0; @@ -120,7 +124,6 @@ static ulonglong start_position, stop_position; static char *start_datetime_str, *stop_datetime_str; static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX; static ulonglong rec_count= 0; -static short binlog_flags = 0; static MYSQL* mysql = NULL; static const char* dirname_for_local_load= 0; static bool opt_skip_annotate_row_events= 0; @@ -142,7 +145,9 @@ enum Exit_status { /** An error occurred and execution should stop. */ ERROR_STOP, /** No error occurred but execution should stop. */ - OK_STOP + OK_STOP, + /** No error occurred - end of file reached. */ + OK_EOF, }; /** @@ -1368,8 +1373,14 @@ static struct my_option my_options[] = {"read-from-remote-server", 'R', "Read binary logs from a MySQL server.", &remote_opt, &remote_opt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"result-file", 'r', "Direct output to a given file.", 0, 0, 0, GET_STR, - REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"raw", 0, "Requires -R. Output raw binlog data instead of SQL " + "statements. Output files named after server logs.", + &opt_raw_mode, &opt_raw_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, + 0, 0}, + {"result-file", 'r', "Direct output to a given file. With --raw this is a " + "prefix for the file names.", + &result_file_name, &result_file_name, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, {"server-id", 0, "Extract only binlog entries created by the server having the given id.", &server_id, &server_id, 0, GET_ULONG, @@ -1418,6 +1429,14 @@ static struct my_option my_options[] = "(you should probably use quotes for your shell to set it properly).", &stop_datetime_str, &stop_datetime_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"stop-never", 0, "Wait for more data from the server " + "instead of stopping at the end of the last log. Implies --to-last-log.", + &opt_stop_never, &opt_stop_never, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"stop-never-slave-server-id", 0, + "The slave server_id used for --read-from-remote-server --stop-never.", + &opt_stop_never_slave_server_id, &opt_stop_never_slave_server_id, 0, + GET_ULONG, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"stop-position", OPT_STOP_POSITION, "Stop reading the binlog at position N. Applies to the last binlog " "passed on the command line.", @@ -1620,10 +1639,6 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), else tty_password=1; break; - case 'r': - if (!(result_file = my_fopen(argument, O_WRONLY | O_BINARY, MYF(MY_WME)))) - exit(1); - break; case 'R': remote_opt= 1; break; @@ -1719,7 +1734,6 @@ static int parse_args(int *argc, char*** argv) { int ho_error; - result_file = stdout; if ((ho_error=handle_options(argc, argv, my_options, get_one_option))) exit(ho_error); if (debug_info_flag) @@ -1809,7 +1823,8 @@ static Exit_status dump_log_entries(const char* logname) Set safe delimiter, to dump things like CREATE PROCEDURE safely */ - fprintf(result_file, "DELIMITER /*!*/;\n"); + if (!opt_raw_mode) + fprintf(result_file, "DELIMITER /*!*/;\n"); strmov(print_event_info.delimiter, "/*!*/;"); print_event_info.verbose= short_form ? 0 : verbose; @@ -1818,7 +1833,8 @@ static Exit_status dump_log_entries(const char* logname) dump_local_log_entries(&print_event_info, logname)); /* Set delimiter back to semicolon */ - fprintf(result_file, "DELIMITER ;\n"); + if (!opt_raw_mode) + fprintf(result_file, "DELIMITER ;\n"); strmov(print_event_info.delimiter, ";"); return rc; } @@ -1924,6 +1940,247 @@ err: } +static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info, + ulong *len, + const char* logname, + uint logname_len, my_off_t old_off) +{ + const char *error_msg; + Log_event *ev; + NET *net= &mysql->net; + DBUG_ENTER("handle_event_text_mode"); + + if (net->read_pos[5] == ANNOTATE_ROWS_EVENT) + { + if (!(ev= read_remote_annotate_event(net->read_pos + 1, *len - 1, + &error_msg))) + { + error("Could not construct annotate event object: %s", error_msg); + DBUG_RETURN(ERROR_STOP); + } + } + else + { + if (!(ev= Log_event::read_log_event((const char*) net->read_pos + 1 , + *len - 1, &error_msg, + glob_description_event, + opt_verify_binlog_checksum))) + { + error("Could not construct log event object: %s", error_msg); + DBUG_RETURN(ERROR_STOP); + } + /* + If reading from a remote host, ensure the temp_buf for the + Log_event class is pointing to the incoming stream. + */ + ev->register_temp_buf((char *) net->read_pos + 1, FALSE); + } + + Log_event_type type= ev->get_type_code(); + if (glob_description_event->binlog_version >= 3 || + (type != LOAD_EVENT && type != CREATE_FILE_EVENT)) + { + /* + If this is a Rotate event, maybe it's the end of the requested binlog; + in this case we are done (stop transfer). + This is suitable for binlogs, not relay logs (but for now we don't read + relay logs remotely because the server is not able to do that). If one + day we read relay logs remotely, then we will have a problem with the + detection below: relay logs contain Rotate events which are about the + binlogs, so which would trigger the end-detection below. + */ + if (type == ROTATE_EVENT) + { + Rotate_log_event *rev= (Rotate_log_event *)ev; + /* + If this is a fake Rotate event, and not about our log, we can stop + transfer. If this a real Rotate event (so it's not about our log, + it's in our log describing the next log), we print it (because it's + part of our log) and then we will stop when we receive the fake one + soon. + */ + if (rev->when == 0) + { + *len= 1; // fake Rotate, so don't increment old_off + if (!to_last_remote_log) + { + if ((rev->ident_len != logname_len) || + memcmp(rev->new_log_ident, logname, logname_len)) + { + delete ev; + DBUG_RETURN(OK_EOF); + } + /* + Otherwise, this is a fake Rotate for our log, at the very + beginning for sure. Skip it, because it was not in the original + log. If we are running with to_last_remote_log, we print it, + because it serves as a useful marker between binlogs then. + */ + delete ev; + DBUG_RETURN(OK_CONTINUE); + } + } + } + else if (type == FORMAT_DESCRIPTION_EVENT) + { + /* + This could be an fake Format_description_log_event that server + (5.0+) automatically sends to a slave on connect, before sending + a first event at the requested position. If this is the case, + don't increment old_off. Real Format_description_log_event always + starts from BIN_LOG_HEADER_SIZE position. + */ + if (old_off != BIN_LOG_HEADER_SIZE) + *len= 1; // fake event, don't increment old_off + } + Exit_status retval= process_event(print_event_info, ev, old_off, logname); + if (retval != OK_CONTINUE) + DBUG_RETURN(retval); + } + else + { + Load_log_event *le= (Load_log_event*)ev; + const char *old_fname= le->fname; + uint old_len= le->fname_len; + File file; + Exit_status retval; + char fname[FN_REFLEN+1]; + + if ((file= load_processor.prepare_new_file_for_old_format(le,fname)) < 0) + { + DBUG_RETURN(ERROR_STOP); + } + + retval= process_event(print_event_info, ev, old_off, logname); + if (retval != OK_CONTINUE) + { + my_close(file,MYF(MY_WME)); + DBUG_RETURN(retval); + } + retval= load_processor.load_old_format_file(net,old_fname,old_len,file); + my_close(file,MYF(MY_WME)); + if (retval != OK_CONTINUE) + DBUG_RETURN(retval); + } + + DBUG_RETURN(OK_CONTINUE); +} + + +static char out_file_name[FN_REFLEN + 1]; + +static Exit_status handle_event_raw_mode(PRINT_EVENT_INFO *print_event_info, + ulong *len, + const char* logname, uint logname_len) +{ + const char *error_msg; + const unsigned char *read_pos= mysql->net.read_pos + 1; + Log_event_type type; + DBUG_ENTER("handle_event_raw_mode"); + DBUG_ASSERT(opt_raw_mode && remote_opt); + + type= (Log_event_type) read_pos[EVENT_TYPE_OFFSET]; + + if (type == HEARTBEAT_LOG_EVENT) + DBUG_RETURN(OK_CONTINUE); + + if (type == ROTATE_EVENT || type == FORMAT_DESCRIPTION_EVENT) + { + Log_event *ev; + if (!(ev= Log_event::read_log_event((const char*) read_pos , + *len - 1, &error_msg, + glob_description_event, + opt_verify_binlog_checksum))) + { + error("Could not construct %s event object: %s", + type == ROTATE_EVENT ? "rotate" : "format description", error_msg); + DBUG_RETURN(ERROR_STOP); + } + /* + If reading from a remote host, ensure the temp_buf for the + Log_event class is pointing to the incoming stream. + */ + ev->register_temp_buf((char *) read_pos, FALSE); + + if (type == ROTATE_EVENT) + { + Exit_status ret_val= OK_CONTINUE; + Rotate_log_event *rev= (Rotate_log_event *)ev; + char *pe= strmake(out_file_name, output_prefix, sizeof(out_file_name)-1); + strmake(pe, rev->new_log_ident, sizeof(out_file_name) - (pe-out_file_name)); + + /* + If this is a fake Rotate event, and not about our log, we can stop + transfer. If this a real Rotate event (so it's not about our log, + it's in our log describing the next log), we print it (because it's + part of our log) and then we will stop when we receive the fake one + soon. + */ + if (rev->when == 0) + { + if (!to_last_remote_log) + { + if ((rev->ident_len != logname_len) || + memcmp(rev->new_log_ident, logname, logname_len)) + { + ret_val= OK_EOF; + } + /* + Otherwise, this is a fake Rotate for our log, at the very + beginning for sure. Skip it, because it was not in the original + log. If we are running with to_last_remote_log, we print it, + because it serves as a useful marker between binlogs then. + */ + } + *len= 1; // fake Rotate, so don't increment old_off + ev->temp_buf= 0; + delete ev; + DBUG_RETURN(ret_val); + } + ev->temp_buf= 0; + delete ev; + } + else /* if (type == FORMAT_DESCRIPTION_EVENT) */ + { + DBUG_ASSERT(type == FORMAT_DESCRIPTION_EVENT); + + if (result_file) + my_fclose(result_file, MYF(0)); + + if (!(result_file= my_fopen(out_file_name, + O_WRONLY | O_BINARY, MYF(MY_WME)))) + { + error("Could not create output log file: %s", out_file_name); + DBUG_RETURN(ERROR_STOP); + } + /* TODO - add write error simulation here */ + + if (my_fwrite(result_file, (const uchar *) BINLOG_MAGIC, + BIN_LOG_HEADER_SIZE, MYF(MY_NABP))) + { + error("Could not write into log file '%s'", out_file_name); + DBUG_RETURN(ERROR_STOP); + } + + delete glob_description_event; + glob_description_event= (Format_description_log_event*) ev; + print_event_info->common_header_len= + glob_description_event->common_header_len; + ev->temp_buf= 0; + /* We do not want to delete the event here. */ + } + } + + if (my_fwrite(result_file, read_pos, *len - 1, MYF(MY_NABP))) + { + error("Could not write into log file '%s'", out_file_name); + DBUG_RETURN(ERROR_STOP); + } + + DBUG_RETURN(OK_CONTINUE); +} + + /** Requests binlog dump from a remote server and prints the events it receives. @@ -1946,8 +2203,9 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info, uint logname_len; NET* net; my_off_t old_off= start_position_mot; - char fname[FN_REFLEN+1]; Exit_status retval= OK_CONTINUE; + short binlog_flags = 0; + ulong slave_id; DBUG_ENTER("dump_remote_log_entries"); /* @@ -1970,6 +2228,9 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info, int4store(buf, (uint32)start_position); if (!opt_skip_annotate_row_events) binlog_flags|= BINLOG_SEND_ANNOTATE_ROWS_EVENT; + if (!opt_stop_never) + binlog_flags|= BINLOG_DUMP_NON_BLOCK; + int2store(buf + BIN_LOG_HEADER_SIZE, binlog_flags); size_t tlen = strlen(logname); @@ -1979,7 +2240,15 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info, DBUG_RETURN(ERROR_STOP); } logname_len = (uint) tlen; - int4store(buf + 6, 0); + if (opt_stop_never) + { + DBUG_ASSERT(to_last_remote_log); + slave_id= (opt_stop_never_slave_server_id == 0) ? + 1 : opt_stop_never_slave_server_id; + } + else + slave_id= 0; + int4store(buf + 6, slave_id); memcpy(buf + 10, logname, logname_len); if (simple_command(mysql, COM_BINLOG_DUMP, buf, logname_len + 10, 1)) { @@ -1989,9 +2258,6 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info, for (;;) { - const char *error_msg; - Log_event *ev; - len= cli_safe_read(mysql); if (len == packet_error) { @@ -2002,117 +2268,23 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info, break; // end of data DBUG_PRINT("info",( "len: %lu net->read_pos[5]: %d\n", len, net->read_pos[5])); - if (net->read_pos[5] == ANNOTATE_ROWS_EVENT) + if (opt_raw_mode) { - if (!(ev= read_remote_annotate_event(net->read_pos + 1, len - 1, - &error_msg))) - { - error("Could not construct annotate event object: %s", error_msg); - DBUG_RETURN(ERROR_STOP); - } + retval= handle_event_raw_mode(print_event_info, &len, + logname, logname_len); } else { - if (!(ev= Log_event::read_log_event((const char*) net->read_pos + 1 , - len - 1, &error_msg, - glob_description_event, - opt_verify_binlog_checksum))) - { - error("Could not construct log event object: %s", error_msg); - DBUG_RETURN(ERROR_STOP); - } - /* - If reading from a remote host, ensure the temp_buf for the - Log_event class is pointing to the incoming stream. - */ - ev->register_temp_buf((char *) net->read_pos + 1, FALSE); + retval= handle_event_text_mode(print_event_info, &len, + logname, logname_len, old_off); } - - Log_event_type type= ev->get_type_code(); - if (glob_description_event->binlog_version >= 3 || - (type != LOAD_EVENT && type != CREATE_FILE_EVENT)) + if (retval != OK_CONTINUE) { - /* - If this is a Rotate event, maybe it's the end of the requested binlog; - in this case we are done (stop transfer). - This is suitable for binlogs, not relay logs (but for now we don't read - relay logs remotely because the server is not able to do that). If one - day we read relay logs remotely, then we will have a problem with the - detection below: relay logs contain Rotate events which are about the - binlogs, so which would trigger the end-detection below. - */ - if (type == ROTATE_EVENT) - { - Rotate_log_event *rev= (Rotate_log_event *)ev; - /* - If this is a fake Rotate event, and not about our log, we can stop - transfer. If this a real Rotate event (so it's not about our log, - it's in our log describing the next log), we print it (because it's - part of our log) and then we will stop when we receive the fake one - soon. - */ - if (rev->when == 0) - { - if (!to_last_remote_log) - { - if ((rev->ident_len != logname_len) || - memcmp(rev->new_log_ident, logname, logname_len)) - { - delete ev; - DBUG_RETURN(OK_CONTINUE); - } - /* - Otherwise, this is a fake Rotate for our log, at the very - beginning for sure. Skip it, because it was not in the original - log. If we are running with to_last_remote_log, we print it, - because it serves as a useful marker between binlogs then. - */ - delete ev; - continue; - } - len= 1; // fake Rotate, so don't increment old_off - } - } - else if (type == FORMAT_DESCRIPTION_EVENT) - { - /* - This could be an fake Format_description_log_event that server - (5.0+) automatically sends to a slave on connect, before sending - a first event at the requested position. If this is the case, - don't increment old_off. Real Format_description_log_event always - starts from BIN_LOG_HEADER_SIZE position. - */ - if (old_off != BIN_LOG_HEADER_SIZE) - len= 1; // fake event, don't increment old_off - } - Exit_status retval= process_event(print_event_info, ev, old_off, logname); - if (retval != OK_CONTINUE) - DBUG_RETURN(retval); + if (retval == OK_EOF) + break; + DBUG_RETURN(retval); } - else - { - Load_log_event *le= (Load_log_event*)ev; - const char *old_fname= le->fname; - uint old_len= le->fname_len; - File file; - Exit_status retval; - if ((file= load_processor.prepare_new_file_for_old_format(le,fname)) < 0) - { - DBUG_RETURN(ERROR_STOP); - } - - retval= process_event(print_event_info, ev, old_off, logname); - if (retval != OK_CONTINUE) - { - my_close(file,MYF(MY_WME)); - DBUG_RETURN(retval); - } - retval= load_processor.load_old_format_file(net,old_fname,old_len,file); - my_close(file,MYF(MY_WME)); - if (retval != OK_CONTINUE) - DBUG_RETURN(retval); - } /* Let's adjust offset for remote log as for local log to produce similar text and to have --stop-position to work identically. @@ -2499,6 +2671,43 @@ int main(int argc, char** argv) my_set_max_open_files(open_files_limit); + if (opt_stop_never) + to_last_remote_log= TRUE; + + if (opt_raw_mode) + { + if (!remote_opt) + { + error("The --raw mode only works with --read-from-remote-server"); + exit(1); + } + if (one_database) + warning("The --database option is ignored in raw mode"); + + if (stop_position != (ulonglong)(~(my_off_t)0)) + warning("The --stop-position option is ignored in raw mode"); + + if (stop_datetime != MY_TIME_T_MAX) + warning("The --stop-datetime option is ignored in raw mode"); + result_file= 0; + if (result_file_name) + output_prefix= result_file_name; + } + else + { + if (result_file_name) + { + if (!(result_file= my_fopen(result_file_name, + O_WRONLY | O_BINARY, MYF(MY_WME)))) + { + error("Could not create log file '%s'", result_file_name); + exit(1); + } + } + else + result_file= stdout; + } + MY_TMPDIR tmpdir; tmpdir.list= 0; if (!dirname_for_local_load) @@ -2521,29 +2730,32 @@ int main(int argc, char** argv) else load_processor.init_by_cur_dir(); - fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;\n"); + if (!opt_raw_mode) + { + fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;\n"); - fprintf(result_file, - "/*!40019 SET @@session.max_insert_delayed_threads=0*/;\n"); - - if (disable_log_bin) fprintf(result_file, - "/*!32316 SET @OLD_SQL_LOG_BIN=@@SQL_LOG_BIN, SQL_LOG_BIN=0*/;\n"); + "/*!40019 SET @@session.max_insert_delayed_threads=0*/;\n"); - /* - In mysqlbinlog|mysql, don't want mysql to be disconnected after each - transaction (which would be the case with GLOBAL.COMPLETION_TYPE==2). - */ - fprintf(result_file, - "/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE," - "COMPLETION_TYPE=0*/;\n"); + if (disable_log_bin) + fprintf(result_file, + "/*!32316 SET @OLD_SQL_LOG_BIN=@@SQL_LOG_BIN, SQL_LOG_BIN=0*/;\n"); - if (charset) + /* + In mysqlbinlog|mysql, don't want mysql to be disconnected after each + transaction (which would be the case with GLOBAL.COMPLETION_TYPE==2). + */ fprintf(result_file, - "\n/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;" - "\n/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;" - "\n/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;" - "\n/*!40101 SET NAMES %s */;\n", charset); + "/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE," + "COMPLETION_TYPE=0*/;\n"); + + if (charset) + fprintf(result_file, + "\n/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;" + "\n/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;" + "\n/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;" + "\n/*!40101 SET NAMES %s */;\n", charset); + } for (save_stop_position= stop_position, stop_position= ~(my_off_t)0 ; (--argc >= 0) ; ) @@ -2557,27 +2769,30 @@ int main(int argc, char** argv) start_position= BIN_LOG_HEADER_SIZE; } - /* - Issue a ROLLBACK in case the last printed binlog was crashed and had half - of transaction. - */ - fprintf(result_file, - "# End of log file\nROLLBACK /* added by mysqlbinlog */;\n" - "/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;\n"); - if (disable_log_bin) - fprintf(result_file, "/*!32316 SET SQL_LOG_BIN=@OLD_SQL_LOG_BIN*/;\n"); - - if (charset) + if (!opt_raw_mode) + { + /* + Issue a ROLLBACK in case the last printed binlog was crashed and had half + of transaction. + */ fprintf(result_file, - "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n" - "/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;\n" - "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n"); + "# End of log file\nROLLBACK /* added by mysqlbinlog */;\n" + "/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;\n"); + if (disable_log_bin) + fprintf(result_file, "/*!32316 SET SQL_LOG_BIN=@OLD_SQL_LOG_BIN*/;\n"); - fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;\n"); + if (charset) + fprintf(result_file, + "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n" + "/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;\n" + "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n"); + + fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;\n"); + } if (tmpdir.list) free_tmpdir(&tmpdir); - if (result_file != stdout) + if (result_file && result_file != stdout) my_fclose(result_file, MYF(0)); cleanup(); free_annotate_event(); diff --git a/mysql-test/r/mysqlbinlog_raw_mode.result b/mysql-test/r/mysqlbinlog_raw_mode.result new file mode 100644 index 00000000000..b64a2148a82 --- /dev/null +++ b/mysql-test/r/mysqlbinlog_raw_mode.result @@ -0,0 +1,274 @@ +reset master; +set timestamp=1000000000; +drop table if exists t1; +CREATE TABLE t1 (c01 BIT); +INSERT INTO t1 VALUES (0); +INSERT INTO t1 VALUES (1); +DROP TABLE t1; +CREATE TABLE t1 (c01 BIT(7)); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (4); +INSERT INTO t1 VALUES (8); +INSERT INTO t1 VALUES (16); +INSERT INTO t1 VALUES (32); +INSERT INTO t1 VALUES (64); +INSERT INTO t1 VALUES (127); +DELETE FROM t1 WHERE c01=127; +UPDATE t1 SET c01=15 WHERE c01=16; +DROP TABLE t1; +CREATE TABLE t1 (a BIT(20), b CHAR(2)); +INSERT INTO t1 VALUES (b'00010010010010001001', 'ab'); +DROP TABLE t1; +CREATE TABLE t1 (c02 BIT(64)); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (128); +INSERT INTO t1 VALUES (b'1111111111111111111111111111111111111111111111111111111111111111'); +DROP TABLE t1; +CREATE TABLE t1 (c03 TINYINT); +INSERT INTO t1 VALUES (1),(2),(3); +INSERT INTO t1 VALUES (-128); +UPDATE t1 SET c03=2 WHERE c03=1; +DELETE FROM t1 WHERE c03=-128; +DROP TABLE t1; +CREATE TABLE t1 (c04 TINYINT UNSIGNED); +INSERT INTO t1 VALUES (128), (255); +UPDATE t1 SET c04=2 WHERE c04=1; +DELETE FROM t1 WHERE c04=255; +DROP TABLE t1; +CREATE TABLE t1 (c06 BOOL); +INSERT INTO t1 VALUES (TRUE); +DELETE FROM t1 WHERE c06=TRUE; +DROP TABLE t1; +CREATE TABLE t1 (c07 SMALLINT); +INSERT INTO t1 VALUES (1234); +DELETE FROM t1 WHERE c07=1234; +DROP TABLE t1; +CREATE TABLE t1 (c08 SMALLINT UNSIGNED); +INSERT INTO t1 VALUES (32768), (65535); +UPDATE t1 SET c08=2 WHERE c08=32768; +DELETE FROM t1 WHERE c08=65535; +DROP TABLE t1; +CREATE TABLE t1 (c10 MEDIUMINT); +INSERT INTO t1 VALUES (12345); +DELETE FROM t1 WHERE c10=12345; +DROP TABLE t1; +CREATE TABLE t1 (c11 MEDIUMINT UNSIGNED); +INSERT INTO t1 VALUES (8388608), (16777215); +UPDATE t1 SET c11=2 WHERE c11=8388608; +DELETE FROM t1 WHERE c11=16777215; +DROP TABLE t1; +CREATE TABLE t1 (c13 INT); +INSERT INTO t1 VALUES (123456); +DELETE FROM t1 WHERE c13=123456; +DROP TABLE t1; +CREATE TABLE t1 (c14 INT UNSIGNED); +INSERT INTO t1 VALUES (2147483648), (4294967295); +UPDATE t1 SET c14=2 WHERE c14=2147483648; +DELETE FROM t1 WHERE c14=4294967295; +DROP TABLE t1; +CREATE TABLE t1 (c16 BIGINT); +INSERT INTO t1 VALUES (1234567890); +DELETE FROM t1 WHERE c16=1234567890; +DROP TABLE t1; +CREATE TABLE t1 (c17 BIGINT UNSIGNED); +INSERT INTO t1 VALUES (9223372036854775808), (18446744073709551615); +UPDATE t1 SET c17=2 WHERE c17=9223372036854775808; +DELETE FROM t1 WHERE c17=18446744073709551615; +DROP TABLE t1; +CREATE TABLE t1 (c19 FLOAT); +INSERT INTO t1 VALUES (123.2234); +DELETE FROM t1 WHERE c19>123; +DROP TABLE t1; +CREATE TABLE t1 (c22 DOUBLE); +INSERT INTO t1 VALUES (123434.22344545); +DELETE FROM t1 WHERE c22>123434; +DROP TABLE t1; +CREATE TABLE t1 (c25 DECIMAL(10,5)); +INSERT INTO t1 VALUES (124.45); +INSERT INTO t1 VALUES (-543.21); +DELETE FROM t1 WHERE c25=124.45; +DROP TABLE t1; +CREATE TABLE t1 (c28 DATE); +INSERT INTO t1 VALUES ('2001-02-03'); +DELETE FROM t1 WHERE c28='2001-02-03'; +DROP TABLE t1; +CREATE TABLE t1 (c29 DATETIME); +INSERT INTO t1 VALUES ('2001-02-03 10:20:30'); +DELETE FROM t1 WHERE c29='2001-02-03 10:20:30'; +DROP TABLE t1; +CREATE TABLE t1 (c30 TIMESTAMP); +INSERT INTO t1 VALUES ('2001-02-03 10:20:30'); +DELETE FROM t1 WHERE c30='2001-02-03 10:20:30'; +DROP TABLE t1; +CREATE TABLE t1 (c31 TIME); +INSERT INTO t1 VALUES ('11:22:33'); +DELETE FROM t1 WHERE c31='11:22:33'; +DROP TABLE t1; +CREATE TABLE t1 (c32 YEAR); +INSERT INTO t1 VALUES ('2001'); +DELETE FROM t1 WHERE c32=2001; +DROP TABLE t1; +CREATE TABLE t1 (c33 CHAR); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c33='a'; +DROP TABLE t1; +CREATE TABLE t1 (c34 CHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c34=''; +DROP TABLE t1; +CREATE TABLE t1 (c35 CHAR(1)); +INSERT INTO t1 VALUES ('b'); +DELETE FROM t1 WHERE c35='b'; +DROP TABLE t1; +CREATE TABLE t1 (c36 CHAR(255)); +INSERT INTO t1 VALUES (repeat('c',255)); +DELETE FROM t1 WHERE c36>'c'; +DROP TABLE t1; +CREATE TABLE t1 (c37 NATIONAL CHAR); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c37='a'; +DROP TABLE t1; +CREATE TABLE t1 (c38 NATIONAL CHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c38=''; +DROP TABLE t1; +CREATE TABLE t1 (c39 NATIONAL CHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c39='a'; +DROP TABLE t1; +CREATE TABLE t1 (c40 NATIONAL CHAR(255)); +INSERT INTO t1 VALUES (repeat('a', 255)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 255)); +DELETE FROM t1 WHERE c40>'a'; +DROP TABLE t1; +CREATE TABLE t1 (c45 VARCHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c45=''; +DROP TABLE t1; +CREATE TABLE t1 (c46 VARCHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c46='a'; +DROP TABLE t1; +CREATE TABLE t1 (c47 VARCHAR(255)); +INSERT INTO t1 VALUES (repeat('a',255)); +DELETE FROM t1 WHERE c47>'a'; +DROP TABLE t1; +CREATE TABLE t1 (c48 VARCHAR(261)); +INSERT INTO t1 VALUES (repeat('a',261)); +DELETE FROM t1 WHERE c48>'a'; +DROP TABLE t1; +CREATE TABLE t1 (c49 NATIONAL VARCHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c49=''; +DROP TABLE t1; +CREATE TABLE t1 (c50 NATIONAL VARCHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c50='a'; +DROP TABLE t1; +CREATE TABLE t1 (c51 NATIONAL VARCHAR(255)); +INSERT INTO t1 VALUES (repeat('a',255)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 255)); +DELETE FROM t1 WHERE c51>'a'; +DROP TABLE t1; +CREATE TABLE t1 (c52 NATIONAL VARCHAR(261)); +INSERT INTO t1 VALUES (repeat('a',261)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 261)); +DELETE FROM t1 WHERE c52>'a'; +DROP TABLE t1; +CREATE TABLE t1 (c57 BINARY); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c57='a'; +DROP TABLE t1; +CREATE TABLE t1 (c58 BINARY(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c58=''; +DROP TABLE t1; +CREATE TABLE t1 (c59 BINARY(1)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c59='a'; +DROP TABLE t1; +CREATE TABLE t1 (c60 BINARY(255)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES (repeat('a\0',120)); +DELETE FROM t1 WHERE c60<0x02; +DROP TABLE t1; +CREATE TABLE t1 (c61 VARBINARY(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c61=''; +DROP TABLE t1; +CREATE TABLE t1 (c62 VARBINARY(1)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c62=0x02; +DROP TABLE t1; +CREATE TABLE t1 (c63 VARBINARY(255)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES (repeat('a\0',120)); +DELETE FROM t1 WHERE c63=0x02; +DROP TABLE t1; +flush logs; +CREATE TABLE t1 (c65 TINYBLOB); +INSERT INTO t1 VALUES ('tinyblob1'); +DELETE FROM t1 WHERE c65='tinyblob1'; +DROP TABLE t1; +CREATE TABLE t1 (c68 BLOB); +INSERT INTO t1 VALUES ('blob1'); +DELETE FROM t1 WHERE c68='blob1'; +DROP TABLE t1; +CREATE TABLE t1 (c71 MEDIUMBLOB); +INSERT INTO t1 VALUES ('mediumblob1'); +DELETE FROM t1 WHERE c71='mediumblob1'; +DROP TABLE t1; +CREATE TABLE t1 (c74 LONGBLOB); +INSERT INTO t1 VALUES ('longblob1'); +DELETE FROM t1 WHERE c74='longblob1'; +DROP TABLE t1; +CREATE TABLE t1 (c66 TINYTEXT); +INSERT INTO t1 VALUES ('tinytext1'); +DELETE FROM t1 WHERE c66='tinytext1'; +DROP TABLE t1; +CREATE TABLE t1 (c69 TEXT); +INSERT INTO t1 VALUES ('text1'); +DELETE FROM t1 WHERE c69='text1'; +DROP TABLE t1; +CREATE TABLE t1 (c72 MEDIUMTEXT); +INSERT INTO t1 VALUES ('mediumtext1'); +DELETE FROM t1 WHERE c72='mediumtext1'; +DROP TABLE t1; +CREATE TABLE t1 (c75 LONGTEXT); +INSERT INTO t1 VALUES ('longtext1'); +DELETE FROM t1 WHERE c75='longtext1'; +DROP TABLE t1; +CREATE TABLE t1 (c77 ENUM('a','b','c')); +INSERT INTO t1 VALUES ('b'); +DELETE FROM t1 WHERE c77='b'; +DROP TABLE t1; +CREATE TABLE t1 (c78 SET('a','b','c','d','e','f')); +INSERT INTO t1 VALUES ('a,b'); +INSERT INTO t1 VALUES ('a,c'); +INSERT INTO t1 VALUES ('b,c'); +INSERT INTO t1 VALUES ('a,b,c'); +INSERT INTO t1 VALUES ('a,b,c,d'); +INSERT INTO t1 VALUES ('a,b,c,d,e'); +INSERT INTO t1 VALUES ('a,b,c,d,e,f'); +DELETE FROM t1 WHERE c78='a,b'; +DROP TABLE t1; +CREATE TABLE t1 (a int NOT NULL DEFAULT 0, b int NOT NULL DEFAULT 0); +CREATE TABLE t2 (a int NOT NULL DEFAULT 0, b int NOT NULL DEFAULT 0); +INSERT INTO t1 SET a=1; +INSERT INTO t1 SET b=1; +INSERT INTO t2 SET a=1; +INSERT INTO t2 SET b=1; +UPDATE t1, t2 SET t1.a=10, t2.a=20; +DROP TABLE t1,t2; +flush logs; +End of tests diff --git a/mysql-test/t/mysqlbinlog_raw_mode.test b/mysql-test/t/mysqlbinlog_raw_mode.test new file mode 100644 index 00000000000..26fb31516f6 --- /dev/null +++ b/mysql-test/t/mysqlbinlog_raw_mode.test @@ -0,0 +1,387 @@ +--source include/have_log_bin.inc +reset master; + +# we need this for getting fixed timestamps inside of this test +set timestamp=1000000000; + +--disable_warnings +drop table if exists t1; +--enable_warnings +CREATE TABLE t1 (c01 BIT); +INSERT INTO t1 VALUES (0); +INSERT INTO t1 VALUES (1); +DROP TABLE t1; + +CREATE TABLE t1 (c01 BIT(7)); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (4); +INSERT INTO t1 VALUES (8); +INSERT INTO t1 VALUES (16); +INSERT INTO t1 VALUES (32); +INSERT INTO t1 VALUES (64); +INSERT INTO t1 VALUES (127); +DELETE FROM t1 WHERE c01=127; +UPDATE t1 SET c01=15 WHERE c01=16; +DROP TABLE t1; + +CREATE TABLE t1 (a BIT(20), b CHAR(2)); +INSERT INTO t1 VALUES (b'00010010010010001001', 'ab'); +DROP TABLE t1; + +CREATE TABLE t1 (c02 BIT(64)); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (128); +INSERT INTO t1 VALUES (b'1111111111111111111111111111111111111111111111111111111111111111'); +DROP TABLE t1; + + +CREATE TABLE t1 (c03 TINYINT); +INSERT INTO t1 VALUES (1),(2),(3); +INSERT INTO t1 VALUES (-128); +UPDATE t1 SET c03=2 WHERE c03=1; +DELETE FROM t1 WHERE c03=-128; +DROP TABLE t1; + +CREATE TABLE t1 (c04 TINYINT UNSIGNED); +INSERT INTO t1 VALUES (128), (255); +UPDATE t1 SET c04=2 WHERE c04=1; +DELETE FROM t1 WHERE c04=255; +DROP TABLE t1; + +CREATE TABLE t1 (c06 BOOL); +INSERT INTO t1 VALUES (TRUE); +DELETE FROM t1 WHERE c06=TRUE; +DROP TABLE t1; + +CREATE TABLE t1 (c07 SMALLINT); +INSERT INTO t1 VALUES (1234); +DELETE FROM t1 WHERE c07=1234; +DROP TABLE t1; + +CREATE TABLE t1 (c08 SMALLINT UNSIGNED); +INSERT INTO t1 VALUES (32768), (65535); +UPDATE t1 SET c08=2 WHERE c08=32768; +DELETE FROM t1 WHERE c08=65535; +DROP TABLE t1; + +CREATE TABLE t1 (c10 MEDIUMINT); +INSERT INTO t1 VALUES (12345); +DELETE FROM t1 WHERE c10=12345; +DROP TABLE t1; + +CREATE TABLE t1 (c11 MEDIUMINT UNSIGNED); +INSERT INTO t1 VALUES (8388608), (16777215); +UPDATE t1 SET c11=2 WHERE c11=8388608; +DELETE FROM t1 WHERE c11=16777215; +DROP TABLE t1; + +CREATE TABLE t1 (c13 INT); +INSERT INTO t1 VALUES (123456); +DELETE FROM t1 WHERE c13=123456; +DROP TABLE t1; + +CREATE TABLE t1 (c14 INT UNSIGNED); +INSERT INTO t1 VALUES (2147483648), (4294967295); +UPDATE t1 SET c14=2 WHERE c14=2147483648; +DELETE FROM t1 WHERE c14=4294967295; +DROP TABLE t1; + +CREATE TABLE t1 (c16 BIGINT); +INSERT INTO t1 VALUES (1234567890); +DELETE FROM t1 WHERE c16=1234567890; +DROP TABLE t1; + +CREATE TABLE t1 (c17 BIGINT UNSIGNED); +INSERT INTO t1 VALUES (9223372036854775808), (18446744073709551615); +UPDATE t1 SET c17=2 WHERE c17=9223372036854775808; +DELETE FROM t1 WHERE c17=18446744073709551615; +DROP TABLE t1; + +CREATE TABLE t1 (c19 FLOAT); +INSERT INTO t1 VALUES (123.2234); +DELETE FROM t1 WHERE c19>123; +DROP TABLE t1; + +CREATE TABLE t1 (c22 DOUBLE); +INSERT INTO t1 VALUES (123434.22344545); +DELETE FROM t1 WHERE c22>123434; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c25 DECIMAL(10,5)); +INSERT INTO t1 VALUES (124.45); +INSERT INTO t1 VALUES (-543.21); +DELETE FROM t1 WHERE c25=124.45; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c28 DATE); +INSERT INTO t1 VALUES ('2001-02-03'); +DELETE FROM t1 WHERE c28='2001-02-03'; +DROP TABLE t1; + +CREATE TABLE t1 (c29 DATETIME); +INSERT INTO t1 VALUES ('2001-02-03 10:20:30'); +DELETE FROM t1 WHERE c29='2001-02-03 10:20:30'; +DROP TABLE t1; + +CREATE TABLE t1 (c30 TIMESTAMP); +INSERT INTO t1 VALUES ('2001-02-03 10:20:30'); +DELETE FROM t1 WHERE c30='2001-02-03 10:20:30'; +DROP TABLE t1; + +CREATE TABLE t1 (c31 TIME); +INSERT INTO t1 VALUES ('11:22:33'); +DELETE FROM t1 WHERE c31='11:22:33'; +DROP TABLE t1; + +CREATE TABLE t1 (c32 YEAR); +INSERT INTO t1 VALUES ('2001'); +DELETE FROM t1 WHERE c32=2001; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c33 CHAR); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c33='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c34 CHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c34=''; +DROP TABLE t1; + +CREATE TABLE t1 (c35 CHAR(1)); +INSERT INTO t1 VALUES ('b'); +DELETE FROM t1 WHERE c35='b'; +DROP TABLE t1; + +CREATE TABLE t1 (c36 CHAR(255)); +INSERT INTO t1 VALUES (repeat('c',255)); +DELETE FROM t1 WHERE c36>'c'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c37 NATIONAL CHAR); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c37='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c38 NATIONAL CHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c38=''; +DROP TABLE t1; + +CREATE TABLE t1 (c39 NATIONAL CHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c39='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c40 NATIONAL CHAR(255)); +INSERT INTO t1 VALUES (repeat('a', 255)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 255)); +DELETE FROM t1 WHERE c40>'a'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c45 VARCHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c45=''; +DROP TABLE t1; + +CREATE TABLE t1 (c46 VARCHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c46='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c47 VARCHAR(255)); +INSERT INTO t1 VALUES (repeat('a',255)); +DELETE FROM t1 WHERE c47>'a'; +DROP TABLE t1; + +CREATE TABLE t1 (c48 VARCHAR(261)); +INSERT INTO t1 VALUES (repeat('a',261)); +DELETE FROM t1 WHERE c48>'a'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c49 NATIONAL VARCHAR(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c49=''; +DROP TABLE t1; + +CREATE TABLE t1 (c50 NATIONAL VARCHAR(1)); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c50='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c51 NATIONAL VARCHAR(255)); +INSERT INTO t1 VALUES (repeat('a',255)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 255)); +DELETE FROM t1 WHERE c51>'a'; +DROP TABLE t1; + +CREATE TABLE t1 (c52 NATIONAL VARCHAR(261)); +INSERT INTO t1 VALUES (repeat('a',261)); +INSERT INTO t1 VALUES (repeat(_latin1 0xDF, 261)); +DELETE FROM t1 WHERE c52>'a'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c57 BINARY); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c57='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c58 BINARY(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c58=''; +DROP TABLE t1; + +CREATE TABLE t1 (c59 BINARY(1)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c59='a'; +DROP TABLE t1; + +CREATE TABLE t1 (c60 BINARY(255)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES (repeat('a\0',120)); +DELETE FROM t1 WHERE c60<0x02; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c61 VARBINARY(0)); +INSERT INTO t1 VALUES (''); +DELETE FROM t1 WHERE c61=''; +DROP TABLE t1; + +CREATE TABLE t1 (c62 VARBINARY(1)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES ('a'); +DELETE FROM t1 WHERE c62=0x02; +DROP TABLE t1; + +CREATE TABLE t1 (c63 VARBINARY(255)); +INSERT INTO t1 VALUES (0x00); +INSERT INTO t1 VALUES (0x02); +INSERT INTO t1 VALUES (repeat('a\0',120)); +DELETE FROM t1 WHERE c63=0x02; +DROP TABLE t1; + +# +flush logs; + +CREATE TABLE t1 (c65 TINYBLOB); +INSERT INTO t1 VALUES ('tinyblob1'); +DELETE FROM t1 WHERE c65='tinyblob1'; +DROP TABLE t1; + +CREATE TABLE t1 (c68 BLOB); +INSERT INTO t1 VALUES ('blob1'); +DELETE FROM t1 WHERE c68='blob1'; +DROP TABLE t1; + +CREATE TABLE t1 (c71 MEDIUMBLOB); +INSERT INTO t1 VALUES ('mediumblob1'); +DELETE FROM t1 WHERE c71='mediumblob1'; +DROP TABLE t1; + +CREATE TABLE t1 (c74 LONGBLOB); +INSERT INTO t1 VALUES ('longblob1'); +DELETE FROM t1 WHERE c74='longblob1'; +DROP TABLE t1; + +CREATE TABLE t1 (c66 TINYTEXT); +INSERT INTO t1 VALUES ('tinytext1'); +DELETE FROM t1 WHERE c66='tinytext1'; +DROP TABLE t1; + +CREATE TABLE t1 (c69 TEXT); +INSERT INTO t1 VALUES ('text1'); +DELETE FROM t1 WHERE c69='text1'; +DROP TABLE t1; + +CREATE TABLE t1 (c72 MEDIUMTEXT); +INSERT INTO t1 VALUES ('mediumtext1'); +DELETE FROM t1 WHERE c72='mediumtext1'; +DROP TABLE t1; + +CREATE TABLE t1 (c75 LONGTEXT); +INSERT INTO t1 VALUES ('longtext1'); +DELETE FROM t1 WHERE c75='longtext1'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c77 ENUM('a','b','c')); +INSERT INTO t1 VALUES ('b'); +DELETE FROM t1 WHERE c77='b'; +DROP TABLE t1; + +# + +CREATE TABLE t1 (c78 SET('a','b','c','d','e','f')); +INSERT INTO t1 VALUES ('a,b'); +INSERT INTO t1 VALUES ('a,c'); +INSERT INTO t1 VALUES ('b,c'); +INSERT INTO t1 VALUES ('a,b,c'); +INSERT INTO t1 VALUES ('a,b,c,d'); +INSERT INTO t1 VALUES ('a,b,c,d,e'); +INSERT INTO t1 VALUES ('a,b,c,d,e,f'); +DELETE FROM t1 WHERE c78='a,b'; +DROP TABLE t1; + +# +# Check multi-table update +# +CREATE TABLE t1 (a int NOT NULL DEFAULT 0, b int NOT NULL DEFAULT 0); +CREATE TABLE t2 (a int NOT NULL DEFAULT 0, b int NOT NULL DEFAULT 0); +INSERT INTO t1 SET a=1; +INSERT INTO t1 SET b=1; +INSERT INTO t2 SET a=1; +INSERT INTO t2 SET b=1; +UPDATE t1, t2 SET t1.a=10, t2.a=20; +DROP TABLE t1,t2; + +flush logs; + +let $MYSQLD_DATADIR= `select @@datadir`; + +# Test reading one file in raw mode +--exec $MYSQL_BINLOG --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT --result-file=$MYSQLTEST_VARDIR/tmp/ master-bin.000001 +--diff_files $MYSQLTEST_VARDIR/tmp/master-bin.000001 $MYSQLD_DATADIR/master-bin.000001 +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000001 + +# Test reading all files in raw mode +# Don't test the end file since this is still open with mysqld so will be different +--exec $MYSQL_BINLOG --raw --read-from-remote-server --to-last-log --user=root --host=127.0.0.1 --port=$MASTER_MYPORT --result-file=$MYSQLTEST_VARDIR/tmp/ master-bin.000001 +--diff_files $MYSQLTEST_VARDIR/tmp/master-bin.000001 $MYSQLD_DATADIR/master-bin.000001 +--diff_files $MYSQLTEST_VARDIR/tmp/master-bin.000002 $MYSQLD_DATADIR/master-bin.000002 + +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000001 +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000002 +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000003 + + +# Test output to different filename +--exec $MYSQL_BINLOG --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT --result-file=$MYSQLTEST_VARDIR/tmp/server1- master-bin.000001 +--diff_files $MYSQLTEST_VARDIR/tmp/server1-master-bin.000001 $MYSQLD_DATADIR/master-bin.000001 +--remove_file $MYSQLTEST_VARDIR/tmp/server1-master-bin.000001 + +--echo End of tests diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 4d62f72f571..db7149953f3 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -320,10 +320,11 @@ /* Used to check GROUP BY list in the MODE_ONLY_FULL_GROUP_BY mode */ #define UNDEF_POS (-1) +#endif /* !MYSQL_CLIENT */ + /* BINLOG_DUMP options */ #define BINLOG_DUMP_NON_BLOCK 1 -#endif /* !MYSQL_CLIENT */ #define BINLOG_SEND_ANNOTATE_ROWS_EVENT 2 From 071ae303bd7ad6ce71f05a25695cd0e9fcef112c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 27 Apr 2016 11:07:43 +0300 Subject: [PATCH 16/24] MDEV-9121: innodb_force_recovery = 6 cannot recover ANY DATA when change buffer not empty Fix: Allow not empty change buffer when innodb_force_recovery >= 5 and output only a warning to error log. Note: Before using force recovery you should always take backup of your database. --- storage/innobase/dict/dict0boot.cc | 20 +++++++++++++++----- storage/xtradb/dict/dict0boot.cc | 20 +++++++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/storage/innobase/dict/dict0boot.cc b/storage/innobase/dict/dict0boot.cc index 573357b54ee..0dff05567d7 100644 --- a/storage/innobase/dict/dict0boot.cc +++ b/storage/innobase/dict/dict0boot.cc @@ -458,12 +458,22 @@ dict_boot(void) if (err == DB_SUCCESS) { if (srv_read_only_mode && !ibuf_is_empty()) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Change buffer must be empty when --innodb-read-only " - "is set!"); + if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Change buffer must be empty when --innodb-read-only " + "is set! " + "You can try to recover the database with innodb_force_recovery=5"); - err = DB_ERROR; - } else { + err = DB_ERROR; + } else { + ib_logf(IB_LOG_LEVEL_WARN, + "Change buffer not empty when --innodb-read-only " + "is set! but srv_force_recovery = %d, ignoring.", + srv_force_recovery); + } + } + + if (err == DB_SUCCESS) { /* Load definitions of other indexes on system tables */ dict_load_sys_table(dict_sys->sys_tables); diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index 0a21264e23d..59f210fcab9 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -464,12 +464,22 @@ dict_boot(void) if (err == DB_SUCCESS) { if (srv_read_only_mode && !ibuf_is_empty()) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Change buffer must be empty when --innodb-read-only " - "is set!"); + if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Change buffer must be empty when --innodb-read-only " + "is set!" + "You can try to recover the database with innodb_force_recovery=5"); - err = DB_ERROR; - } else { + err = DB_ERROR; + } else { + ib_logf(IB_LOG_LEVEL_WARN, + "Change buffer not empty when --innodb-read-only " + "is set! but srv_force_recovery = %d, ignoring.", + srv_force_recovery); + } + } + + if (err == DB_SUCCESS) { /* Load definitions of other indexes on system tables */ dict_load_sys_table(dict_sys->sys_tables); From 646c4cea58afbb369021a3d7b5ccbbf6aed708d4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 27 Apr 2016 16:13:14 +0200 Subject: [PATCH 17/24] MDEV-9973 : Do not set permissions for serviceaccount user (Win7 and later) This appears to break some installation, and it did not do anything useful anyway. --- win/packaging/extra.wxs.in | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index 3425a76427b..978a5c9c90d 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -542,17 +542,6 @@ Value="utf8" /> - - - 600)]]> - - - - - - From 48f02af761a8ae575388820ddb73622aaf109175 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 27 Apr 2016 16:37:01 +0300 Subject: [PATCH 18/24] MDEV-9602 crash in st_key::actual_rec_per_key when group by constant Problem was that cost_group_min_max() could not handle if group by was optimized away. --- mysql-test/r/group_by.result | 14 ++++++++++++++ mysql-test/t/group_by.test | 10 ++++++++++ sql/opt_range.cc | 12 +++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result index 07773960e5a..3ef73bb1943 100644 --- a/mysql-test/r/group_by.result +++ b/mysql-test/r/group_by.result @@ -2678,3 +2678,17 @@ NULL 100098 100099 drop table t0,t1,t2; +# +# MDEV-9602 crash in st_key::actual_rec_per_key when group by constant +# +create table t1 (a date not null,unique (a)) engine=innodb; +Warnings: +Warning 1286 Unknown storage engine 'innodb' +Warning 1266 Using storage engine MyISAM for table 't1' +select distinct a from t1 group by 'a'; +a +insert into t1 values("2001-02-02"),("2001-02-03"); +select distinct a from t1 group by 'a'; +a +2001-02-02 +drop table t1; diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test index fe9308cd20a..5d8b79b9fca 100644 --- a/mysql-test/t/group_by.test +++ b/mysql-test/t/group_by.test @@ -1793,3 +1793,13 @@ from t1 group by t1.b; drop table t0,t1,t2; + +--echo # +--echo # MDEV-9602 crash in st_key::actual_rec_per_key when group by constant +--echo # + +create table t1 (a date not null,unique (a)) engine=innodb; +select distinct a from t1 group by 'a'; +insert into t1 values("2001-02-02"),("2001-02-03"); +select distinct a from t1 group by 'a'; +drop table t1; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index ed7e9a56ae5..ea5f77eed1d 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -13105,7 +13105,17 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, num_blocks= (ha_rows)(table_records / keys_per_block) + 1; /* Compute the number of keys in a group. */ - keys_per_group= (ha_rows) index_info->actual_rec_per_key(group_key_parts - 1); + if (!group_key_parts) + { + /* Summary over the whole table */ + keys_per_group= table_records; + } + else + { + keys_per_group= (ha_rows) index_info->actual_rec_per_key(group_key_parts - + 1); + } + if (keys_per_group == 0) /* If there is no statistics try to guess */ /* each group contains 10% of all records */ keys_per_group= (table_records / 10) + 1; From 7c6cb41b15624fa77cbc92c7b5de9a834e011d56 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 27 Apr 2016 16:38:24 +0300 Subject: [PATCH 19/24] Ignore files in tmp (like mysqld.S) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4c7efe9a119..f2064537da4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ Docs/INFO_SRC Makefile TAGS Testing/ +tmp/ VERSION.dep configure client/async_example From b4ff64568c88ab3ce559e7bd39853d9cbf86704a Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 28 Apr 2016 11:10:55 +0300 Subject: [PATCH 20/24] Fixed wrong counting of global Memory_used --- include/mysql/plugin.h | 4 +- include/mysql/plugin_audit.h.pp | 3 +- include/mysql/plugin_auth.h.pp | 3 +- include/mysql/plugin_encryption.h.pp | 3 +- include/mysql/plugin_ftparser.h.pp | 3 +- include/mysql/plugin_password_validation.h.pp | 3 +- libmysqld/lib_sql.cc | 3 + sql/mysqld.cc | 85 +++++++++++-------- sql/sql_class.cc | 16 +++- sql/sql_class.h | 16 ++++ sql/sql_show.cc | 21 +++-- sql/sql_show.h | 2 +- 12 files changed, 111 insertions(+), 51 deletions(-) diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index cfa4b13a7ef..b3c71c65488 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -194,8 +194,10 @@ struct st_mysql_show_var { enum enum_mysql_show_type type; }; +struct system_status_var; + #define SHOW_VAR_FUNC_BUFF_SIZE (256 * sizeof(void*)) -typedef int (*mysql_show_var_func)(MYSQL_THD, struct st_mysql_show_var*, void *, enum enum_var_type); +typedef int (*mysql_show_var_func)(MYSQL_THD, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); /* diff --git a/include/mysql/plugin_audit.h.pp b/include/mysql/plugin_audit.h.pp index f4df16afa67..24f2c69345d 100644 --- a/include/mysql/plugin_audit.h.pp +++ b/include/mysql/plugin_audit.h.pp @@ -281,7 +281,8 @@ struct st_mysql_show_var { void *value; enum enum_mysql_show_type type; }; -typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, enum enum_var_type); +struct system_status_var; +typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); struct st_mysql_sys_var; struct st_mysql_value; typedef int (*mysql_var_check_func)(void* thd, diff --git a/include/mysql/plugin_auth.h.pp b/include/mysql/plugin_auth.h.pp index 046f92b5ab8..10cd10bf9c8 100644 --- a/include/mysql/plugin_auth.h.pp +++ b/include/mysql/plugin_auth.h.pp @@ -281,7 +281,8 @@ struct st_mysql_show_var { void *value; enum enum_mysql_show_type type; }; -typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, enum enum_var_type); +struct system_status_var; +typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); struct st_mysql_sys_var; struct st_mysql_value; typedef int (*mysql_var_check_func)(void* thd, diff --git a/include/mysql/plugin_encryption.h.pp b/include/mysql/plugin_encryption.h.pp index 850dbf05a58..46d3c3d5a55 100644 --- a/include/mysql/plugin_encryption.h.pp +++ b/include/mysql/plugin_encryption.h.pp @@ -281,7 +281,8 @@ struct st_mysql_show_var { void *value; enum enum_mysql_show_type type; }; -typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, enum enum_var_type); +struct system_status_var; +typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); struct st_mysql_sys_var; struct st_mysql_value; typedef int (*mysql_var_check_func)(void* thd, diff --git a/include/mysql/plugin_ftparser.h.pp b/include/mysql/plugin_ftparser.h.pp index ee1056a36d7..17de800875e 100644 --- a/include/mysql/plugin_ftparser.h.pp +++ b/include/mysql/plugin_ftparser.h.pp @@ -281,7 +281,8 @@ struct st_mysql_show_var { void *value; enum enum_mysql_show_type type; }; -typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, enum enum_var_type); +struct system_status_var; +typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); struct st_mysql_sys_var; struct st_mysql_value; typedef int (*mysql_var_check_func)(void* thd, diff --git a/include/mysql/plugin_password_validation.h.pp b/include/mysql/plugin_password_validation.h.pp index 0f14ac1eb53..1abdbd30f57 100644 --- a/include/mysql/plugin_password_validation.h.pp +++ b/include/mysql/plugin_password_validation.h.pp @@ -281,7 +281,8 @@ struct st_mysql_show_var { void *value; enum enum_mysql_show_type type; }; -typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, enum enum_var_type); +struct system_status_var; +typedef int (*mysql_show_var_func)(void*, struct st_mysql_show_var*, void *, struct system_status_var *status_var, enum enum_var_type); struct st_mysql_sys_var; struct st_mysql_value; typedef int (*mysql_var_check_func)(void* thd, diff --git a/libmysqld/lib_sql.cc b/libmysqld/lib_sql.cc index 86a0676d97e..a313c624078 100644 --- a/libmysqld/lib_sql.cc +++ b/libmysqld/lib_sql.cc @@ -516,6 +516,9 @@ int init_embedded_server(int argc, char **argv, char **groups) if (my_thread_init()) return 1; + if (init_early_variables()) + return 1; + if (argc) { argcp= &argc; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ced922536fc..cb4f95212ef 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3997,35 +3997,31 @@ extern "C" { static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific) { THD *thd= current_thd; - /* If thread specific memory */ - if (likely(is_thread_specific)) + + if (likely(is_thread_specific)) /* If thread specific memory */ { - if (mysqld_server_initialized || thd) - { - /* - THD may not be set if we are called from my_net_init() before THD - thread has started. - However, this should never happen, so better to assert and - fix this. - */ - DBUG_ASSERT(thd); - if (thd) - { - DBUG_PRINT("info", ("memory_used: %lld size: %lld", - (longlong) thd->status_var.local_memory_used, - size)); - thd->status_var.local_memory_used+= size; - DBUG_ASSERT((longlong) thd->status_var.local_memory_used >= 0); - } - } + /* + When thread specfic is set, both mysqld_server_initialized and thd + must be set + */ + DBUG_ASSERT(mysqld_server_initialized && thd); + + DBUG_PRINT("info", ("thd memory_used: %lld size: %lld", + (longlong) thd->status_var.local_memory_used, + size)); + thd->status_var.local_memory_used+= size; + DBUG_ASSERT((longlong) thd->status_var.local_memory_used >= 0); } else if (likely(thd)) + { + DBUG_PRINT("info", ("global thd memory_used: %lld size: %lld", + (longlong) thd->status_var.global_memory_used, + size)); thd->status_var.global_memory_used+= size; + } else { - // workaround for gcc 4.2.4-1ubuntu4 -fPIE (from DEB_BUILD_HARDENING=1) - int64 volatile * volatile ptr=&global_status_var.global_memory_used; - my_atomic_add64_explicit(ptr, size, MY_MEMORY_ORDER_RELAXED); + update_global_memory_status(size); } } } @@ -4066,6 +4062,22 @@ rpl_make_log_name(const char *opt, DBUG_RETURN(NULL); } +/* We have to setup my_malloc_size_cb_func early to catch all mallocs */ + +static int init_early_variables() +{ + if (pthread_key_create(&THR_THD, NULL)) + { + fprintf(stderr, "Fatal error: Can't create thread-keys\n"); + return 1; + } + set_current_thd(0); + set_malloc_size_cb(my_malloc_size_cb_func); + global_status_var.global_memory_used= 0; + return 0; +} + + static int init_common_variables() { umask(((~my_umask) & 0666)); @@ -4077,16 +4089,12 @@ static int init_common_variables() connection_errors_peer_addr= 0; my_decimal_set_zero(&decimal_zero); // set decimal_zero constant; - if (pthread_key_create(&THR_THD,NULL) || - pthread_key_create(&THR_MALLOC,NULL)) + if (pthread_key_create(&THR_MALLOC,NULL)) { sql_print_error("Can't create thread-keys"); return 1; } - set_current_thd(0); - set_malloc_size_cb(my_malloc_size_cb_func); - init_libstrings(); tzset(); // Set tzname @@ -5184,6 +5192,9 @@ static int init_server_components() init_global_table_stats(); init_global_index_stats(); + /* It's now safe to use thread specific memory */ + mysqld_server_initialized= 1; + /* Allow storage engine to give real error messages */ if (ha_init_errors()) DBUG_RETURN(1); @@ -5544,6 +5555,9 @@ int mysqld_main(int argc, char **argv) sf_leaking_memory= 1; // no safemalloc memory leak reports if we exit early mysqld_server_started= mysqld_server_initialized= 0; + if (init_early_variables()) + exit(1); + #ifdef HAVE_NPTL ld_assume_kernel_is_set= (getenv("LD_ASSUME_KERNEL") != 0); #endif @@ -5846,9 +5860,6 @@ int mysqld_main(int argc, char **argv) if (Events::init((THD*) 0, opt_noacl || opt_bootstrap)) unireg_abort(1); - /* It's now safe to use thread specific memory */ - mysqld_server_initialized= 1; - if (WSREP_ON) { if (opt_bootstrap) @@ -8260,15 +8271,16 @@ static int show_default_keycache(THD *thd, SHOW_VAR *var, char *buff, static int show_memory_used(THD *thd, SHOW_VAR *var, char *buff, + struct system_status_var *status_var, enum enum_var_type scope) { var->type= SHOW_LONGLONG; var->value= buff; if (scope == OPT_GLOBAL) - *(longlong*) buff= (global_status_var.local_memory_used + - global_status_var.global_memory_used); + *(longlong*) buff= (status_var->global_memory_used + + status_var->local_memory_used); else - *(longlong*) buff= thd->status_var.local_memory_used; + *(longlong*) buff= status_var->local_memory_used; return 0; } @@ -8697,7 +8709,9 @@ static int mysql_init_variables(void) prepared_stmt_count= 0; mysqld_unix_port= opt_mysql_tmpdir= my_bind_addr_str= NullS; bzero((uchar*) &mysql_tmpdir_list, sizeof(mysql_tmpdir_list)); - bzero((char *) &global_status_var, sizeof(global_status_var)); + /* Clear all except global_memory_used */ + bzero((char*) &global_status_var, offsetof(STATUS_VAR, + last_cleared_system_status_var)); opt_large_pages= 0; opt_super_large_pages= 0; #if defined(ENABLED_DEBUG_SYNC) @@ -9931,6 +9945,7 @@ void refresh_status(THD *thd) /* Reset thread's status variables */ thd->set_status_var_init(); + thd->status_var.global_memory_used= 0; bzero((uchar*) &thd->org_status_var, sizeof(thd->org_status_var)); thd->start_bytes_received= 0; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index d2ebbc64c47..236b68586d5 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -914,7 +914,8 @@ THD::THD(bool is_wsrep_applier) */ THD *old_THR_THD= current_thd; set_current_thd(this); - status_var.local_memory_used= status_var.global_memory_used= 0; + status_var.local_memory_used= sizeof(THD); + status_var.global_memory_used= 0; main_da.init(); /* @@ -1636,6 +1637,8 @@ THD::~THD() that memory allocation counting is done correctly */ set_current_thd(this); + if (!status_in_global) + add_status_to_global(); /* Ensure that no one is using THD */ mysql_mutex_lock(&LOCK_thd_data); @@ -1698,13 +1701,14 @@ THD::~THD() if (xid_hash_pins) lf_hash_put_pins(xid_hash_pins); /* Ensure everything is freed */ + status_var.local_memory_used-= sizeof(THD); if (status_var.local_memory_used != 0) { DBUG_PRINT("error", ("memory_used: %lld", status_var.local_memory_used)); SAFEMALLOC_REPORT_MEMORY(my_thread_dbug_id()); DBUG_ASSERT(status_var.local_memory_used == 0); } - + update_global_memory_status(status_var.global_memory_used); set_current_thd(orig_thd == this ? 0 : orig_thd); DBUG_VOID_RETURN; } @@ -1723,6 +1727,7 @@ THD::~THD() other types are handled explicitely */ + void add_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var) { ulong *end= (ulong*) ((uchar*) to_var + @@ -1742,12 +1747,17 @@ void add_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var) to_var->binlog_bytes_written+= from_var->binlog_bytes_written; to_var->cpu_time+= from_var->cpu_time; to_var->busy_time+= from_var->busy_time; - to_var->local_memory_used+= from_var->local_memory_used; /* Update global_memory_used. We have to do this with atomic_add as the global value can change outside of LOCK_status. */ + if (to_var == &global_status_var) + { + DBUG_PRINT("info", ("global memory_used: %lld size: %lld", + (longlong) global_status_var.global_memory_used, + (longlong) from_var->global_memory_used)); + } // workaround for gcc 4.2.4-1ubuntu4 -fPIE (from DEB_BUILD_HARDENING=1) int64 volatile * volatile ptr= &to_var->global_memory_used; my_atomic_add64_explicit(ptr, from_var->global_memory_used, diff --git a/sql/sql_class.h b/sql/sql_class.h index 0e23a5e5916..e1127f77c09 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -822,6 +822,20 @@ void add_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var); void add_diff_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var, STATUS_VAR *dec_var); +/* + Update global_memory_used. We have to do this with atomic_add as the + global value can change outside of LOCK_status. +*/ +inline void update_global_memory_status(int64 size) +{ + DBUG_PRINT("info", ("global memory_used: %lld size: %lld", + (longlong) global_status_var.global_memory_used, + size)); + // workaround for gcc 4.2.4-1ubuntu4 -fPIE (from DEB_BUILD_HARDENING=1) + int64 volatile * volatile ptr= &global_status_var.global_memory_used; + my_atomic_add64_explicit(ptr, size, MY_MEMORY_ORDER_RELAXED); +} + /** Get collation by name, send error to client on failure. @param name Collation name @@ -3836,9 +3850,11 @@ public: void add_status_to_global() { + DBUG_ASSERT(status_in_global == 0); mysql_mutex_lock(&LOCK_status); add_to_status(&global_status_var, &status_var); /* Mark that this THD status has already been added in global status */ + status_var.global_memory_used= 0; status_in_global= 1; mysql_mutex_unlock(&LOCK_status); } diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 4a30b68bbc0..1cbad7e9ea0 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2961,8 +2961,7 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond) thread in this thread. However it's better that we notice it eventually than hide it. */ - table->field[12]->store((longlong) (tmp->status_var.local_memory_used + - sizeof(THD)), + table->field[12]->store((longlong) tmp->status_var.local_memory_used, FALSE); table->field[12]->set_notnull(); table->field[13]->store((longlong) tmp->get_examined_row_count(), TRUE); @@ -3258,7 +3257,8 @@ static bool show_status_array(THD *thd, const char *wild, */ for (var=variables; var->type == SHOW_FUNC || var->type == SHOW_SIMPLE_FUNC; var= &tmp) - ((mysql_show_var_func)(var->value))(thd, &tmp, buff, scope); + ((mysql_show_var_func)(var->value))(thd, &tmp, buff, + status_var, scope); SHOW_TYPE show_type=var->type; if (show_type == SHOW_ARRAY) @@ -3390,10 +3390,14 @@ end: DBUG_RETURN(res); } -/* collect status for all running threads */ +/* + collect status for all running threads + Return number of threads used +*/ -void calc_sum_of_all_status(STATUS_VAR *to) +uint calc_sum_of_all_status(STATUS_VAR *to) { + uint count= 0; DBUG_ENTER("calc_sum_of_all_status"); /* Ensure that thread id not killed during loop */ @@ -3404,16 +3408,21 @@ void calc_sum_of_all_status(STATUS_VAR *to) /* Get global values as base */ *to= global_status_var; + to->local_memory_used= 0; /* Add to this status from existing threads */ while ((tmp= it++)) { + count++; if (!tmp->status_in_global) + { add_to_status(to, &tmp->status_var); + to->local_memory_used+= tmp->status_var.local_memory_used; + } } mysql_mutex_unlock(&LOCK_thread_count); - DBUG_VOID_RETURN; + DBUG_RETURN(count); } diff --git a/sql/sql_show.h b/sql/sql_show.h index 9dae78e7f0e..dbae2a42b39 100644 --- a/sql/sql_show.h +++ b/sql/sql_show.h @@ -102,7 +102,7 @@ bool mysqld_show_authors(THD *thd); bool mysqld_show_contributors(THD *thd); bool mysqld_show_privileges(THD *thd); char *make_backup_log_name(char *buff, const char *name, const char* log_ext); -void calc_sum_of_all_status(STATUS_VAR *to); +uint calc_sum_of_all_status(STATUS_VAR *to); void append_definer(THD *thd, String *buffer, const LEX_STRING *definer_user, const LEX_STRING *definer_host); int add_status_vars(SHOW_VAR *list); From 732adec0a4c75d99389230feeb0deca0ad668de7 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 28 Apr 2016 13:39:05 +0300 Subject: [PATCH 21/24] Removed some not needed when doing delete thd, which caused warnings about wrong mutex usage from safe_mutex. Ensure that LOCK_status is always taken before LOCK_thread_count --- sql/log.cc | 2 -- sql/mysqld.cc | 1 + sql/rpl_parallel.cc | 5 +++-- sql/slave.cc | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 6e11f4bbe5d..a938dcbc9f8 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -9597,9 +9597,7 @@ binlog_background_thread(void *arg __attribute__((unused))) THD_STAGE_INFO(thd, stage_binlog_stopping_background_thread); - mysql_mutex_lock(&LOCK_thread_count); delete thd; - mysql_mutex_unlock(&LOCK_thread_count); my_thread_end(); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index cb4f95212ef..404836b4312 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4630,6 +4630,7 @@ static int init_thread_environment() mysql_mutex_init(key_LOCK_global_system_variables, &LOCK_global_system_variables, MY_MUTEX_INIT_FAST); mysql_mutex_record_order(&LOCK_active_mi, &LOCK_global_system_variables); + mysql_mutex_record_order(&LOCK_status, &LOCK_thread_count); mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); mysql_mutex_init(key_LOCK_prepared_stmt_count, diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 697da6fa10e..f2437a6953b 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1372,10 +1372,11 @@ handle_rpl_parallel_thread(void *arg) thd->reset_db(NULL, 0); thd_proc_info(thd, "Slave worker thread exiting"); thd->temporary_tables= 0; + mysql_mutex_lock(&LOCK_thread_count); - THD_CHECK_SENTRY(thd); - delete thd; + thd->unlink(); mysql_mutex_unlock(&LOCK_thread_count); + delete thd; mysql_mutex_lock(&rpt->LOCK_rpl_thread); rpt->running= false; diff --git a/sql/slave.cc b/sql/slave.cc index e9bcfb073b4..5410192a8dd 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -311,9 +311,7 @@ handle_slave_init(void *arg __attribute__((unused))) thd->get_stmt_da()->sql_errno(), thd->get_stmt_da()->message()); - mysql_mutex_lock(&LOCK_thread_count); delete thd; - mysql_mutex_unlock(&LOCK_thread_count); thread_safe_decrement32(&service_thread_count); signal_thd_deleted(); my_thread_end(); @@ -4961,8 +4959,9 @@ err_during_init: mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_thread_count); - delete thd; + thd->unlink(); mysql_mutex_unlock(&LOCK_thread_count); + delete thd; thread_safe_decrement32(&service_thread_count); signal_thd_deleted(); From ea83c1d7c61f6e267dfbbf01781cbbee7835ebe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 28 Apr 2016 15:21:10 +0300 Subject: [PATCH 22/24] MDEV-9977: Crash when accessing large (>4G) InnoDB table on MariaDB 10.1.x 32-bit binaries. Problem was the fact that tablespace size was incorrectly rounded to next extent size (1M). --- storage/innobase/fil/fil0fil.cc | 6 ++---- storage/xtradb/fil/fil0fil.cc | 8 +++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d918b21d779..22abf592adc 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -737,11 +737,9 @@ fil_node_open_file( } } - if (size_bytes >= FSP_EXTENT_SIZE * UNIV_PAGE_SIZE) { + if (size_bytes >= (1024*1024)) { /* Truncate the size to whole extent size. */ - size_bytes = ut_2pow_round(size_bytes, - FSP_EXTENT_SIZE * - UNIV_PAGE_SIZE); + size_bytes = ut_2pow_round(size_bytes, (1024*1024)); } if (!fsp_flags_is_compressed(flags)) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index b60a0e9ddaf..2da234ad094 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -739,11 +739,9 @@ fil_node_open_file( } } - if (size_bytes >= FSP_EXTENT_SIZE * UNIV_PAGE_SIZE) { + if (size_bytes >= (1024*1024)) { /* Truncate the size to whole extent size. */ - size_bytes = ut_2pow_round(size_bytes, - FSP_EXTENT_SIZE * - UNIV_PAGE_SIZE); + size_bytes = ut_2pow_round(size_bytes, (1024*1024)); } if (!fsp_flags_is_compressed(flags)) { @@ -5683,7 +5681,7 @@ fil_space_get_node( /* Found! */ break; } else { - *block_offset -= node->size; + (*block_offset) -= node->size; node = UT_LIST_GET_NEXT(chain, node); } } From d62b7585224034535b1cdfd0b2e0ec0eee881614 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 28 Apr 2016 14:14:09 +0300 Subject: [PATCH 23/24] Moved mysqld_server_initialized to before galera is initialized. --- sql/mysqld.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 404836b4312..a4c6184f313 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5103,6 +5103,9 @@ static int init_server_components() variables even when a wsrep provider is not loaded. */ + /* It's now safe to use thread specific memory */ + mysqld_server_initialized= 1; + wsrep_thr_init(); if (WSREP_ON && !wsrep_recovery && !opt_abort) /* WSREP BEFORE SE */ @@ -5193,9 +5196,6 @@ static int init_server_components() init_global_table_stats(); init_global_index_stats(); - /* It's now safe to use thread specific memory */ - mysqld_server_initialized= 1; - /* Allow storage engine to give real error messages */ if (ha_init_errors()) DBUG_RETURN(1); From d5822a3ad0657040114cdc185c6387b9eb3a12b2 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 28 Apr 2016 16:27:42 +0300 Subject: [PATCH 24/24] Fixed some galera tests --- mysql-test/suite/galera/include/galera_sst_set_mysqldump.inc | 1 + mysql-test/suite/galera/r/galera_ist_mysqldump.result | 1 + mysql-test/suite/galera/r/galera_sst_mysqldump.result | 1 + mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result | 1 + mysql-test/suite/galera/r/mysql-wsrep#237.result | 1 + mysql-test/suite/galera/r/mysql-wsrep#33.result | 1 + mysql-test/suite/galera/t/mysql-wsrep#237.test | 2 ++ 7 files changed, 8 insertions(+) diff --git a/mysql-test/suite/galera/include/galera_sst_set_mysqldump.inc b/mysql-test/suite/galera/include/galera_sst_set_mysqldump.inc index 5f87d23dcc1..cbd2c1c817a 100644 --- a/mysql-test/suite/galera/include/galera_sst_set_mysqldump.inc +++ b/mysql-test/suite/galera/include/galera_sst_set_mysqldump.inc @@ -6,6 +6,7 @@ --connection node_1 # We need a user with a password to perform SST, otherwise we hit LP #1378253 +CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; --let $wsrep_sst_auth_orig = `SELECT @@wsrep_sst_auth` diff --git a/mysql-test/suite/galera/r/galera_ist_mysqldump.result b/mysql-test/suite/galera/r/galera_ist_mysqldump.result index 9a5b4e8a76f..788d60051b5 100644 --- a/mysql-test/suite/galera/r/galera_ist_mysqldump.result +++ b/mysql-test/suite/galera/r/galera_ist_mysqldump.result @@ -1,4 +1,5 @@ Setting SST method to mysqldump ... +CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; SET GLOBAL wsrep_sst_method = 'mysqldump'; diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump.result b/mysql-test/suite/galera/r/galera_sst_mysqldump.result index e35c4055f45..5c0d9a45d41 100644 --- a/mysql-test/suite/galera/r/galera_sst_mysqldump.result +++ b/mysql-test/suite/galera/r/galera_sst_mysqldump.result @@ -1,4 +1,5 @@ Setting SST method to mysqldump ... +CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; SET GLOBAL wsrep_sst_method = 'mysqldump'; diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result index 7d30b356aa9..227e1c15444 100644 --- a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result +++ b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result @@ -1,4 +1,5 @@ Setting SST method to mysqldump ... +CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; SET GLOBAL wsrep_sst_method = 'mysqldump'; diff --git a/mysql-test/suite/galera/r/mysql-wsrep#237.result b/mysql-test/suite/galera/r/mysql-wsrep#237.result index 3fd9aed1480..1889a8feca0 100644 --- a/mysql-test/suite/galera/r/mysql-wsrep#237.result +++ b/mysql-test/suite/galera/r/mysql-wsrep#237.result @@ -8,3 +8,4 @@ SLEEP(1) 0 SET DEBUG_SYNC= 'now SIGNAL continue'; DROP TABLE t; +SET DEBUG_SYNC= 'RESET'; diff --git a/mysql-test/suite/galera/r/mysql-wsrep#33.result b/mysql-test/suite/galera/r/mysql-wsrep#33.result index 62af519ad32..fc647a2000d 100644 --- a/mysql-test/suite/galera/r/mysql-wsrep#33.result +++ b/mysql-test/suite/galera/r/mysql-wsrep#33.result @@ -1,4 +1,5 @@ Setting SST method to mysqldump ... +CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; SET GLOBAL wsrep_sst_method = 'mysqldump'; diff --git a/mysql-test/suite/galera/t/mysql-wsrep#237.test b/mysql-test/suite/galera/t/mysql-wsrep#237.test index 7a65cb52ae9..f2dd6bce711 100644 --- a/mysql-test/suite/galera/t/mysql-wsrep#237.test +++ b/mysql-test/suite/galera/t/mysql-wsrep#237.test @@ -29,3 +29,5 @@ SET DEBUG_SYNC= 'now SIGNAL continue'; --reap DROP TABLE t; +--connection node_1a +SET DEBUG_SYNC= 'RESET';