From 5bf5e6eeca8e1988dd14cee9ea66ec74e67f0d2a Mon Sep 17 00:00:00 2001 From: Dominik Hassler Date: Sat, 12 Nov 2022 13:14:23 +0000 Subject: [PATCH 1/6] OS detection logic in my_gethwaddr.c is backwards --- mysys/my_gethwaddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysys/my_gethwaddr.c b/mysys/my_gethwaddr.c index 24054aa4151..1f344af88fd 100644 --- a/mysys/my_gethwaddr.c +++ b/mysys/my_gethwaddr.c @@ -116,7 +116,7 @@ my_bool my_gethwaddr(uchar *to) uint i; for (i= 0; res && i < ifc.ifc_len / sizeof(ifr[0]); i++) { -#if !defined(_AIX) || !defined(__linux__) +#if defined(_AIX) || defined(__linux__) #if defined(__linux__) #define HWADDR_DATA ifr[i].ifr_hwaddr.sa_data #else From 2283f82de5e6eed14e4378cc084658a4c7c9fe51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Nov 2022 10:22:29 +0200 Subject: [PATCH 2/6] MDEV-29905 fixup: Remove some unnecessary code srv_shutdown(): Do not call log_free_check(), because it will now be repeatedly called by ibuf_merge_all(). Do not call srv_sync_log_buffer_in_background(), because we do not actually care about durability during shutdown. Log writes will already be triggered by buf_flush_page_cleaner() for writing back modified pages, possibly by log_free_check(). logs_empty_and_mark_files_at_shutdown(): Clean up a condition. This function is the caller of srv_shutdown(), and it will ensure that the log and the buffer pool will be in clean state before shutdown. --- storage/innobase/log/log0log.cc | 10 +++------- storage/innobase/srv/srv0srv.cc | 5 ----- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 39d606c120a..73c847479ef 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1096,13 +1096,9 @@ loop: } /* We need these threads to stop early in shutdown. */ - const char* thread_name; - - if (srv_fast_shutdown != 2 && trx_rollback_is_active) { - thread_name = "rollback of recovered transactions"; - } else { - thread_name = NULL; - } + const char* thread_name = srv_fast_shutdown != 2 + && trx_rollback_is_active + ? "rollback of recovered transactions" : nullptr; if (thread_name) { ut_ad(!srv_read_only_mode); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index f8e02f95c5e..4460f4bce0a 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1677,13 +1677,8 @@ void srv_shutdown(bool ibuf_merge) n_tables_to_drop = row_drop_tables_for_mysql_in_background(); if (ibuf_merge) { - srv_main_thread_op_info = "checking free log space"; - log_free_check(); srv_main_thread_op_info = "doing insert buffer merge"; n_bytes_merged = ibuf_merge_all(); - - /* Flush logs if needed */ - srv_sync_log_buffer_in_background(); } /* Print progress message every 60 seconds during shutdown */ From e0e096faaad2aee1717a955c2b4b9100f1ed7749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Nov 2022 12:18:03 +0200 Subject: [PATCH 3/6] MDEV-29982 Improve the InnoDB log overwrite error message The InnoDB write-ahead log ib_logfile0 is of fixed size, specified by innodb_log_file_size. If the tail of the log manages to overwrite the head (latest checkpoint) of the log, crash recovery will be broken. Let us clarify the messages about this, including adding a message on the completion of a log checkpoint that notes that the dangerous situation is over. To reproduce the dangerous scenario, we will introduce the debug injection label ib_log_checkpoint_avoid_hard, which will avoid log checkpoints even harder than the previous ib_log_checkpoint_avoid. log_t::overwrite_warned: The first known dangerous log sequence number. Set in log_close() and cleared in log_write_checkpoint_info(), which will output a "Crash recovery was broken" message. --- .../encryption/r/innochecksum,debug.rdiff | 10 ++++++++ .../suite/encryption/t/innochecksum.test | 14 +++++++++++ storage/innobase/buf/buf0flu.cc | 5 ++++ storage/innobase/include/log0log.h | 7 +++++- storage/innobase/log/log0log.cc | 17 ++++++++++++- storage/innobase/mtr/mtr0mtr.cc | 25 +++++++++++-------- 6 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innochecksum,debug.rdiff diff --git a/mysql-test/suite/encryption/r/innochecksum,debug.rdiff b/mysql-test/suite/encryption/r/innochecksum,debug.rdiff new file mode 100644 index 00000000000..c3e3eed26bd --- /dev/null +++ b/mysql-test/suite/encryption/r/innochecksum,debug.rdiff @@ -0,0 +1,10 @@ +@@ -30,6 +30,9 @@ + # Space ID mismatch + # Restore the original tables + # Corrupt FIL_DATA+10 (data) ++# FOUND 1 is expected for both. ++FOUND 1 /InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size; last checkpoint LSN=\d+, current LSN=\d+\. Shutdown is in progress\..*InnoDB: Crash recovery was broken.*/ in mysqld.1.err ++FOUND 1 /InnoDB: Crash recovery was broken/ in mysqld.1.err + # Run innochecksum on t2 + # Run innochecksum on t3 + # Run innochecksum on t6 diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index 59d90fbb3d7..ecabce30ab7 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -9,6 +9,7 @@ -- source include/have_file_key_management_plugin.inc -- source include/innodb_page_size_small.inc -- source include/innodb_checksum_algorithm.inc +-- source include/maybe_debug.inc if (!$INNOCHECKSUM) { --echo Need innochecksum binary @@ -18,6 +19,10 @@ if (!$INNOCHECKSUM) { --disable_query_log # This may be triggered on a slow system or one that lacks native AIO. call mtr.add_suppression("InnoDB: Trying to delete tablespace.*pending operations"); +if ($have_debug) { +SET GLOBAL DEBUG_DBUG='+d,ib_log_checkpoint_avoid_hard'; +call mtr.add_suppression("InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size"); +} --enable_query_log let $checksum_algorithm = `SELECT @@innodb_checksum_algorithm`; @@ -259,6 +264,15 @@ print FILE pack("H*", "c00lcafedeadb017"); close FILE or die "close"; EOF +if ($have_debug) { +--let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN= InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size; last checkpoint LSN=\\d+, current LSN=\\d+\\. Shutdown is in progress\\..*InnoDB: Crash recovery was broken.* +--echo # FOUND 1 is expected for both. +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= InnoDB: Crash recovery was broken +--source include/search_pattern_in_file.inc +} + -- disable_result_log --error 1 --exec $INNOCHECKSUM $t1_IBD diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 3d2669e2cc5..ee8bdaeb19d 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1947,6 +1947,7 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) write out before we can advance the checkpoint. */ if (sync_lsn > log_sys.get_flushed_lsn()) log_write_up_to(sync_lsn, true); + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", return;); log_checkpoint(); } } @@ -1962,6 +1963,8 @@ ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious) if (recv_recovery_is_on()) recv_sys.apply(true); + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", return;); + Atomic_relaxed &limit= furious ? buf_flush_sync_lsn : buf_flush_async_lsn; @@ -2316,6 +2319,7 @@ unemployed: buf_pool.page_cleaner_set_idle(true); DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", continue;); + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", continue;); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -2399,6 +2403,7 @@ do_checkpoint: here should not affect correctness, because log_free_check() should still be invoking checkpoints when needed. */ DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", goto next;); + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto next;); if (!recv_recovery_is_on() && srv_operation == SRV_OPERATION_NORMAL) log_checkpoint(); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 870f5da0925..4a5567ff62d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, 2021, MariaDB Corporation. +Copyright (c) 2017, 2022, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -460,6 +460,11 @@ public: size_t buf_free; /** recommended maximum size of buf, after which the buffer is flushed */ size_t max_buf_free; + + /** Log sequence number when a log file overwrite (broken crash recovery) + was noticed. Protected by mutex. */ + lsn_t overwrite_warned; + /** mutex to serialize access to the flush list when we are putting dirty blocks in the list. The idea behind this mutex is to be able to release log_sys.mutex during mtr_commit and still ensure that diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 73c847479ef..53183605759 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -53,6 +53,7 @@ Created 12/9/1995 Heikki Tuuri #include "sync0sync.h" #include "buf0dump.h" #include "log0sync.h" +#include "log.h" /* General philosophy of InnoDB redo-logs: @@ -965,11 +966,20 @@ ATTRIBUTE_COLD void log_write_checkpoint_info(lsn_t end_lsn) DBUG_PRINT("ib_log", ("checkpoint ended at " LSN_PF ", flushed to " LSN_PF, - lsn_t{log_sys.last_checkpoint_lsn}, + log_sys.next_checkpoint_lsn, log_sys.get_flushed_lsn())); MONITOR_INC(MONITOR_NUM_CHECKPOINT); + if (log_sys.overwrite_warned) { + sql_print_information("InnoDB: Crash recovery was broken " + "between LSN=" LSN_PF + " and checkpoint LSN=" LSN_PF ".", + log_sys.overwrite_warned, + log_sys.next_checkpoint_lsn); + log_sys.overwrite_warned = 0; + } + mysql_mutex_unlock(&log_sys.mutex); } @@ -997,10 +1007,15 @@ func_exit: const lsn_t sync_lsn= checkpoint + log_sys.max_checkpoint_age; if (lsn <= sync_lsn) { +#ifndef DBUG_OFF + skip_checkpoint: +#endif log_sys.set_check_flush_or_checkpoint(false); goto func_exit; } + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;); + mysql_mutex_unlock(&log_sys.mutex); /* We must wait to prevent the tail of the log overwriting the head. */ diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 651bda06145..0aa5b9a4c3f 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -24,14 +24,15 @@ Mini-transaction buffer Created 11/26/1995 Heikki Tuuri *******************************************************/ -#include "mtr0mtr.h" +#include "mtr0log.h" #include "buf0buf.h" #include "buf0flu.h" #include "fsp0sysspace.h" #include "page0types.h" -#include "mtr0log.h" #include "log0recv.h" +#include "srv0start.h" +#include "log.h" /** Iterate over a memo block in reverse. */ template @@ -759,7 +760,6 @@ mtr_t::memo_release(const void* object, ulint type) static bool log_margin_warned; static time_t log_margin_warn_time; -static bool log_close_warned; static time_t log_close_warn_time; /** Check margin not to overwrite transaction log from the last checkpoint. @@ -797,8 +797,8 @@ static void log_margin_checkpoint_age(ulint len) log_margin_warned= true; log_margin_warn_time= t; - ib::error() << "innodb_log_file_size is too small " - "for mini-transaction size " << len; + sql_print_error("InnoDB: innodb_log_file_size is too small " + "for mini-transaction size " ULINTPF, len); } } else if (UNIV_LIKELY(lsn + margin <= log_sys.last_checkpoint_lsn + @@ -924,14 +924,19 @@ static mtr_t::page_flush_ahead log_close(lsn_t lsn) checkpoint_age != lsn) { time_t t= time(nullptr); - if (!log_close_warned || difftime(t, log_close_warn_time) > 15) + if (!log_sys.overwrite_warned || difftime(t, log_close_warn_time) > 15) { - log_close_warned= true; + if (!log_sys.overwrite_warned) + log_sys.overwrite_warned= lsn; log_close_warn_time= t; - ib::error() << "The age of the last checkpoint is " << checkpoint_age - << ", which exceeds the log capacity " - << log_sys.log_capacity << "."; + sql_print_error("InnoDB: Crash recovery is broken due to" + " insufficient innodb_log_file_size;" + " last checkpoint LSN=" LSN_PF ", current LSN=" LSN_PF + "%s.", + lsn_t{log_sys.last_checkpoint_lsn}, lsn, + srv_shutdown_state != SRV_SHUTDOWN_INITIATED + ? ". Shutdown is in progress" : ""); } } else if (UNIV_LIKELY(checkpoint_age <= log_sys.max_modified_age_async)) From ee7fba17491690341763466c627ae5fada9724d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Nov 2022 14:01:37 +0200 Subject: [PATCH 4/6] MDEV-16264 fixup: Remove unused variables --- storage/innobase/srv/srv0srv.cc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 4460f4bce0a..1d92b3670af 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -395,9 +395,6 @@ ib_mutex_t srv_misc_tmpfile_mutex; /** Temporary file for miscellanous diagnostic output */ FILE* srv_misc_tmpfile; -static ulint srv_main_thread_process_no; -static ulint srv_main_thread_id; - /* The following counts are used by the srv_master_callback. */ /** Iterations of the loop bounded by 'srv_active' label. */ @@ -905,12 +902,7 @@ srv_printf_innodb_monitor( n_reserved); } - fprintf(file, - "Process ID=" ULINTPF - ", Main thread ID=" ULINTPF - ", state: %s\n", - srv_main_thread_process_no, - srv_main_thread_id, + fprintf(file, "Process ID=0, Main thread ID=0, state: %s\n", srv_main_thread_op_info); fprintf(file, "Number of rows inserted " ULINTPF From 744b33c2879ef245d65bbfc0c260c0be7a1bd9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Nov 2022 14:12:20 +0200 Subject: [PATCH 5/6] Cleanup: Remove a useless header file --- storage/innobase/CMakeLists.txt | 3 +-- storage/innobase/include/ibuf0ibuf.h | 32 +++++++++++++++++++++++++- storage/innobase/include/ibuf0ibuf.inl | 31 ------------------------- storage/innobase/include/ibuf0types.h | 31 ------------------------- 4 files changed, 32 insertions(+), 65 deletions(-) delete mode 100644 storage/innobase/include/ibuf0types.h diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index bdef91aa7b9..2e7b99fa228 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -311,8 +311,7 @@ SET(INNOBASE_SOURCES include/hash0hash.h include/ib0mutex.h include/ibuf0ibuf.h - include/ibuf0ibuf.inl/ - include/ibuf0types.h + include/ibuf0ibuf.inl include/lock0iter.h include/lock0lock.h include/lock0lock.inl diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index 8cf0aeaeb99..49d837868c4 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -30,7 +30,6 @@ Created 7/19/1997 Heikki Tuuri #include "mtr0mtr.h" #include "dict0mem.h" #include "fsp0fsp.h" -#include "ibuf0types.h" /** Default value for maximum on-disk size of change buffer in terms of percentage of the buffer pool. */ @@ -61,6 +60,37 @@ enum ibuf_use_t { /** Operations that can currently be buffered. */ extern ulong innodb_change_buffering; +/** Insert buffer struct */ +struct ibuf_t{ + ulint size; /*!< current size of the ibuf index + tree, in pages */ + ulint max_size; /*!< recommended maximum size of the + ibuf index tree, in pages */ + ulint seg_size; /*!< allocated pages of the file + segment containing ibuf header and + tree */ + bool empty; /*!< Protected by the page + latch of the root page of the + insert buffer tree + (FSP_IBUF_TREE_ROOT_PAGE_NO). true + if and only if the insert + buffer tree is empty. */ + ulint free_list_len; /*!< length of the free list */ + ulint height; /*!< tree height */ + dict_index_t* index; /*!< insert buffer index */ + + /** number of pages merged */ + Atomic_counter n_merges; + Atomic_counter n_merged_ops[IBUF_OP_COUNT]; + /*!< number of operations of each type + merged to index pages */ + Atomic_counter n_discarded_ops[IBUF_OP_COUNT]; + /*!< number of operations of each type + discarded without merging due to the + tablespace being deleted or the + index being dropped */ +}; + /** The insert buffer control structure */ extern ibuf_t ibuf; diff --git a/storage/innobase/include/ibuf0ibuf.inl b/storage/innobase/include/ibuf0ibuf.inl index 2c2620511c7..9f4e937f31d 100644 --- a/storage/innobase/include/ibuf0ibuf.inl +++ b/storage/innobase/include/ibuf0ibuf.inl @@ -64,37 +64,6 @@ ibuf_mtr_commit( mtr_commit(mtr); } -/** Insert buffer struct */ -struct ibuf_t{ - ulint size; /*!< current size of the ibuf index - tree, in pages */ - ulint max_size; /*!< recommended maximum size of the - ibuf index tree, in pages */ - ulint seg_size; /*!< allocated pages of the file - segment containing ibuf header and - tree */ - bool empty; /*!< Protected by the page - latch of the root page of the - insert buffer tree - (FSP_IBUF_TREE_ROOT_PAGE_NO). true - if and only if the insert - buffer tree is empty. */ - ulint free_list_len; /*!< length of the free list */ - ulint height; /*!< tree height */ - dict_index_t* index; /*!< insert buffer index */ - - /** number of pages merged */ - Atomic_counter n_merges; - Atomic_counter n_merged_ops[IBUF_OP_COUNT]; - /*!< number of operations of each type - merged to index pages */ - Atomic_counter n_discarded_ops[IBUF_OP_COUNT]; - /*!< number of operations of each type - discarded without merging due to the - tablespace being deleted or the - index being dropped */ -}; - /************************************************************************//** Sets the free bit of the page in the ibuf bitmap. This is done in a separate mini-transaction, hence this operation does not restrict further work to only diff --git a/storage/innobase/include/ibuf0types.h b/storage/innobase/include/ibuf0types.h deleted file mode 100644 index 6b7c47208a0..00000000000 --- a/storage/innobase/include/ibuf0types.h +++ /dev/null @@ -1,31 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1997, 2009, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/ibuf0types.h -Insert buffer global types - -Created 7/29/1997 Heikki Tuuri -*******************************************************/ - -#ifndef ibuf0types_h -#define ibuf0types_h - -struct ibuf_t; - -#endif From dc2741be52913f12f17e04341f17c51875aa5065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Nov 2022 15:40:28 +0200 Subject: [PATCH 6/6] MDEV-29984 innodb_fast_shutdown=0 fails to report change buffer merge progress ibuf.size, ibuf.max_size: Changed the type to Atomic_relaxed in order to fix some (not all) race conditions. ibuf_contract(): Renamed from ibuf_merge_pages(ulint*). ibuf_merge(), ibuf_merge_all(): Removed. srv_shutdown(): Invoke log_free_check() and ibuf_contract(). Even though ibuf_contract() is not writing anything, it will trigger calls of ibuf_merge_or_delete_for_page(), which will write something. Because we cannot invoke log_free_check() at that low level, we must invoke it at the high level. srv_shutdown_print(): Replaces srv_shutdown_print_master_pending(). Report progress and remaining work every 15 seconds. For the change buffer merge, the remaining work is indicated by ibuf.size. --- storage/innobase/ibuf/ibuf0ibuf.cc | 110 ++++----------------------- storage/innobase/include/ibuf0ibuf.h | 10 +-- storage/innobase/srv/srv0srv.cc | 76 +++++++++--------- 3 files changed, 61 insertions(+), 135 deletions(-) diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 31fc317c90b..5ac10b0fb33 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2392,16 +2392,11 @@ done: #endif } -/*********************************************************************//** -Contracts insert buffer trees by reading pages to the buffer pool. +/** Contract the change buffer by reading pages to the buffer pool. @return a lower limit for the combined size in bytes of entries which -will be merged from ibuf trees to the pages read, 0 if ibuf is -empty */ -static -ulint -ibuf_merge_pages( -/*=============*/ - ulint* n_pages) /*!< out: number of pages to which merged */ +will be merged from ibuf trees to the pages read +@retval 0 if ibuf.empty */ +ulint ibuf_contract() { mtr_t mtr; btr_pcur_t pcur; @@ -2409,8 +2404,6 @@ ibuf_merge_pages( uint32_t page_nos[IBUF_MAX_N_PAGES_MERGED]; uint32_t space_ids[IBUF_MAX_N_PAGES_MERGED]; - *n_pages = 0; - ibuf_mtr_start(&mtr); /* Open a cursor to a randomly chosen leaf of the tree, at a random @@ -2438,14 +2431,15 @@ ibuf_merge_pages( return(0); } + ulint n_pages = 0; sum_sizes = ibuf_get_merge_page_nos(TRUE, btr_pcur_get_rec(&pcur), &mtr, space_ids, - page_nos, n_pages); + page_nos, &n_pages); ibuf_mtr_commit(&mtr); btr_pcur_close(&pcur); - ibuf_read_merge_pages(space_ids, page_nos, *n_pages); + ibuf_read_merge_pages(space_ids, page_nos, n_pages); return(sum_sizes + 1); } @@ -2519,73 +2513,6 @@ ibuf_merge_space( return(n_pages); } -/** Contract the change buffer by reading pages to the buffer pool. -@param[out] n_pages number of pages merged -@param[in] sync whether the caller waits for -the issued reads to complete -@return a lower limit for the combined size in bytes of entries which -will be merged from ibuf trees to the pages read, 0 if ibuf is -empty */ -MY_ATTRIBUTE((warn_unused_result)) -static ulint ibuf_merge(ulint* n_pages) -{ - *n_pages = 0; - - /* We perform a dirty read of ibuf.empty, without latching - the insert buffer root page. We trust this dirty read except - when a slow shutdown is being executed. During a slow - shutdown, the insert buffer merge must be completed. */ - - if (ibuf.empty && srv_shutdown_state <= SRV_SHUTDOWN_INITIATED) { - return(0); -#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG - } else if (ibuf_debug) { - return(0); -#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ - } else { - return ibuf_merge_pages(n_pages); - } -} - -/** Contract the change buffer by reading pages to the buffer pool. -@return a lower limit for the combined size in bytes of entries which -will be merged from ibuf trees to the pages read, 0 if ibuf is empty */ -static ulint ibuf_contract() -{ - ulint n_pages; - return ibuf_merge_pages(&n_pages); -} - -/** Contract the change buffer by reading pages to the buffer pool. -@return a lower limit for the combined size in bytes of entries which -will be merged from ibuf trees to the pages read, 0 if ibuf is -empty */ -ulint ibuf_merge_all() -{ -#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG - if (ibuf_debug) { - return(0); - } -#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ - - ulint sum_bytes = 0; - ulint n_pages = srv_io_capacity; - - for (ulint sum_pages = 0; sum_pages < n_pages; ) { - log_free_check(); - ulint n_pag2; - ulint n_bytes = ibuf_merge(&n_pag2); - - if (n_bytes == 0) { - break; - } - - sum_bytes += n_bytes; - } - - return sum_bytes; -} - /*********************************************************************//** Contract insert buffer trees after insert if they are too big. */ UNIV_INLINE @@ -2595,13 +2522,7 @@ ibuf_contract_after_insert( ulint entry_size) /*!< in: size of a record which was inserted into an ibuf tree */ { - /* Perform dirty reads of ibuf.size and ibuf.max_size, to - reduce ibuf_mutex contention. ibuf.max_size remains constant - after ibuf_init_at_db_start(), but ibuf.size should be - protected by ibuf_mutex. Given that ibuf.size fits in a - machine word, this should be OK; at worst we are doing some - excessive ibuf_contract() or occasionally skipping a - ibuf_contract(). */ + /* dirty comparison, to avoid contention on ibuf_mutex */ if (ibuf.size < ibuf.max_size) { return; } @@ -3221,16 +3142,17 @@ ibuf_insert_low( do_merge = FALSE; - /* Perform dirty reads of ibuf.size and ibuf.max_size, to - reduce ibuf_mutex contention. Given that ibuf.max_size and - ibuf.size fit in a machine word, this should be OK; at worst - we are doing some excessive ibuf_contract() or occasionally + /* Perform dirty comparison of ibuf.max_size and ibuf.size to + reduce ibuf_mutex contention. This should be OK; at worst we + are doing some excessive ibuf_contract() or occasionally skipping an ibuf_contract(). */ - if (ibuf.max_size == 0) { + const ulint max_size = ibuf.max_size; + + if (max_size == 0) { return(DB_STRONG_FAIL); } - if (ibuf.size >= ibuf.max_size + IBUF_CONTRACT_DO_NOT_INSERT) { + if (ibuf.size >= max_size + IBUF_CONTRACT_DO_NOT_INSERT) { /* Insert buffer is now too big, contract it but do not try to insert */ @@ -4625,7 +4547,7 @@ ibuf_print( fprintf(file, "Ibuf: size " ULINTPF ", free list len " ULINTPF "," " seg size " ULINTPF ", " ULINTPF " merges\n", - ibuf.size, + ulint{ibuf.size}, ibuf.free_list_len, ibuf.seg_size, ulint{ibuf.n_merges}); diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index 49d837868c4..73be4b0a8e8 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -62,9 +62,9 @@ extern ulong innodb_change_buffering; /** Insert buffer struct */ struct ibuf_t{ - ulint size; /*!< current size of the ibuf index + Atomic_relaxed size; /*!< current size of the ibuf index tree, in pages */ - ulint max_size; /*!< recommended maximum size of the + Atomic_relaxed max_size; /*!< recommended maximum size of the ibuf index tree, in pages */ ulint seg_size; /*!< allocated pages of the file segment containing ibuf header and @@ -371,9 +371,9 @@ void ibuf_delete_for_discarded_space(ulint space); /** Contract the change buffer by reading pages to the buffer pool. @return a lower limit for the combined size in bytes of entries which -will be merged from ibuf trees to the pages read, 0 if ibuf is -empty */ -ulint ibuf_merge_all(); +will be merged from ibuf trees to the pages read +@retval 0 if ibuf.empty */ +ulint ibuf_contract(); /** Contracts insert buffer trees by reading pages referring to space_id to the buffer pool. diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 1d92b3670af..5a01b408a4b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -73,6 +73,7 @@ Created 10/8/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "trx0types.h" #include +#include "log.h" #include /* The following is the maximum allowed duration of a lock wait. */ @@ -1495,38 +1496,42 @@ srv_master_evict_from_table_cache( return(n_tables_evicted); } -/*********************************************************************//** -This function prints progress message every 60 seconds during server -shutdown, for any activities that master thread is pending on. */ -static -void -srv_shutdown_print_master_pending( -/*==============================*/ - time_t* last_print_time, /*!< last time the function - print the message */ - ulint n_tables_to_drop, /*!< number of tables to - be dropped */ - ulint n_bytes_merged) /*!< number of change buffer - just merged */ +/** Report progress during shutdown. +@param last time of last output +@param n_drop number of tables to be dropped +@param n_read number of page reads initiated for change buffer merge */ +static void srv_shutdown_print(time_t &last, ulint n_drop, ulint n_read) { - time_t current_time = time(NULL); + time_t now= time(nullptr); + if (now - last >= 15) + { + last= now; - if (difftime(current_time, *last_print_time) > 60) { - *last_print_time = current_time; + if (n_drop) + { + sql_print_information("InnoDB: Waiting for %zu table(s) to be dropped", + n_drop); +#if defined HAVE_SYSTEMD && !defined EMBEDDED_LIBRARY + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "InnoDB: Waiting for %zu table(s)" + " to be dropped", n_drop); +#endif + return; + } - if (n_tables_to_drop) { - ib::info() << "Waiting for " << n_tables_to_drop - << " table(s) to be dropped"; - } - - /* Check change buffer merge, we only wait for change buffer - merge if it is a slow shutdown */ - if (!srv_fast_shutdown && n_bytes_merged) { - ib::info() << "Waiting for change buffer merge to" - " complete number of bytes of change buffer" - " just merged: " << n_bytes_merged; - } - } + const ulint ibuf_size= ibuf.size; + sql_print_information("Completing change buffer merge;" + " %zu page reads initiated;" + " %zu change buffer pages remain", + n_read, ibuf_size); +#if defined HAVE_SYSTEMD && !defined EMBEDDED_LIBRARY + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "Completing change buffer merge;" + " %zu page reads initiated;" + " %zu change buffer pages remain", + n_read, ibuf_size); +#endif + } } /*********************************************************************//** @@ -1654,7 +1659,7 @@ Complete the shutdown tasks such as background DROP TABLE, and optionally change buffer merge (on innodb_fast_shutdown=0). */ void srv_shutdown(bool ibuf_merge) { - ulint n_bytes_merged = 0; + ulint n_read = 0; ulint n_tables_to_drop; time_t now = time(NULL); @@ -1670,15 +1675,14 @@ void srv_shutdown(bool ibuf_merge) if (ibuf_merge) { srv_main_thread_op_info = "doing insert buffer merge"; - n_bytes_merged = ibuf_merge_all(); + log_free_check(); + n_read = ibuf_contract(); } - /* Print progress message every 60 seconds during shutdown */ - if (srv_print_verbose_log) { - srv_shutdown_print_master_pending( - &now, n_tables_to_drop, n_bytes_merged); + if (n_tables_to_drop || ibuf_merge) { + srv_shutdown_print(now, n_tables_to_drop, n_read); } - } while (n_bytes_merged || n_tables_to_drop); + } while (n_read || n_tables_to_drop); } /** The periodic master task controlling the server. */