From 19cdddf17db22eb19626ffdd2565fe04a347af92 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 6 Jul 2023 13:49:06 +0400 Subject: [PATCH 01/43] A cleanup for MDEV-30932 UBSAN: negation of -X cannot be represented in type .. "mtr --view-protocol func_math" failed because of a too long column names imlicitly generated for the underlying expressions. With --view-protocol they were replaced to "Name_exp_1". Adding column aliases for these expressions. --- mysql-test/main/func_math.result | 8 ++++---- mysql-test/main/func_math.test | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/main/func_math.result b/mysql-test/main/func_math.result index c3d7ee20f50..81d0395fe69 100644 --- a/mysql-test/main/func_math.result +++ b/mysql-test/main/func_math.result @@ -3566,11 +3566,11 @@ DROP TABLE t1; SELECT TRUNCATE(0, -9223372036854775808); TRUNCATE(0, -9223372036854775808) 0 -SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))); -GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))) +SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))) AS col1; +col1 NULL -SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0)); -(GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0)) +SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0)) AS col1; +col1 NULL # # End of 10.4 tests diff --git a/mysql-test/main/func_math.test b/mysql-test/main/func_math.test index 4007f81fbee..9db00a54b80 100644 --- a/mysql-test/main/func_math.test +++ b/mysql-test/main/func_math.test @@ -1888,8 +1888,8 @@ DROP TABLE t1; SELECT TRUNCATE(0, -9223372036854775808); --disable_warnings -SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))); -SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0)); +SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))) AS col1; +SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0)) AS col1; --enable_warnings --echo # From 1bfd3cc457c46b7a5f070f9395146cf815c100d7 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Mon, 3 Jul 2023 16:04:15 +0300 Subject: [PATCH 02/43] MDEV-10962 Deadlock with 3 concurrent DELETEs by unique key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PROBLEM: A deadlock was possible when a transaction tried to "upgrade" an already held Record Lock to Next Key Lock. SOLUTION: This patch is based on observations that: (1) a Next Key Lock is equivalent to Record Lock combined with Gap Lock (2) a GAP Lock never has to wait for any other lock In case we request a Next Key Lock, we check if we already own a Record Lock of equal or stronger mode, and if so, then we change the requested lock type to GAP Lock, which we either already have, or can be granted immediately, as GAP locks don't conflict with any other lock types. (We don't consider Insert Intention Locks a Gap Lock in above statements). The reason of why we don't upgrage Record Lock to Next Key Lock is the following. Imagine a transaction which does something like this: for each row { request lock in LOCK_X|LOCK_REC_NOT_GAP mode request lock in LOCK_S mode } If we upgraded lock from Record Lock to Next Key lock, there would be created only two lock_t structs for each page, one for LOCK_X|LOCK_REC_NOT_GAP mode and one for LOCK_S mode, and then used their bitmaps to mark all records from the same page. The situation would look like this: request lock in LOCK_X|LOCK_REC_NOT_GAP mode on row 1: // -> creates new lock_t for LOCK_X|LOCK_REC_NOT_GAP mode and sets bit for // 1 request lock in LOCK_S mode on row 1: // -> notices that we already have LOCK_X|LOCK_REC_NOT_GAP on the row 1, // so it upgrades it to X request lock in LOCK_X|LOCK_REC_NOT_GAP mode on row 2: // -> creates a new lock_t for LOCK_X|LOCK_REC_NOT_GAP mode (because we // don't have any after we've upgraded!) and sets bit for 2 request lock in LOCK_S mode on row 2: // -> notices that we already have LOCK_X|LOCK_REC_NOT_GAP on the row 2, // so it upgrades it to X ...etc...etc.. Each iteration of the loop creates a new lock_t struct, and in the end we have a lot (one for each record!) of LOCK_X locks, each with single bit set in the bitmap. Soon we run out of space for lock_t structs. If we create LOCK_GAP instead of lock upgrading, the above scenario works like the following: // -> creates new lock_t for LOCK_X|LOCK_REC_NOT_GAP mode and sets bit for // 1 request lock in LOCK_S mode on row 1: // -> notices that we already have LOCK_X|LOCK_REC_NOT_GAP on the row 1, // so it creates LOCK_S|LOCK_GAP only and sets bit for 1 request lock in LOCK_X|LOCK_REC_NOT_GAP mode on row 2: // -> reuses the lock_t for LOCK_X|LOCK_REC_NOT_GAP by setting bit for 2 request lock in LOCK_S mode on row 2: // -> notices that we already have LOCK_X|LOCK_REC_NOT_GAP on the row 2, // so it reuses LOCK_S|LOCK_GAP setting bit for 2 In the end we have just two locks per page, one for each mode: LOCK_X|LOCK_REC_NOT_GAP and LOCK_S|LOCK_GAP. Another benefit of this solution is that it avoids not-entirely const-correct, (and otherwise looking risky) "upgrading". The fix was ported from mysql/mysql-server@bfba840dfa7794b988c59c94658920dbe556075d mysql/mysql-server@75cefdb1f73b8f8ac8e22b10dfb5073adbdfdfb0 Reviewed by: Marko Mäkelä --- .../innodb/r/deadlock_on_lock_upgrade.result | 78 ++++++++++ .../innodb/t/deadlock_on_lock_upgrade.test | 143 ++++++++++++++++++ storage/innobase/include/lock0types.h | 27 ++++ storage/innobase/lock/lock0lock.cc | 71 ++++++++- 4 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/innodb/r/deadlock_on_lock_upgrade.result create mode 100644 mysql-test/suite/innodb/t/deadlock_on_lock_upgrade.test diff --git a/mysql-test/suite/innodb/r/deadlock_on_lock_upgrade.result b/mysql-test/suite/innodb/r/deadlock_on_lock_upgrade.result new file mode 100644 index 00000000000..d015e74c65d --- /dev/null +++ b/mysql-test/suite/innodb/r/deadlock_on_lock_upgrade.result @@ -0,0 +1,78 @@ +# +# Bug #23755664 DEADLOCK WITH 3 CONCURRENT DELETES BY UNIQUE KEY +# +connection default; +CREATE TABLE `t`( +`id` INT, +`a` INT DEFAULT NULL, +PRIMARY KEY(`id`), +UNIQUE KEY `u`(`a`) +) ENGINE=InnoDB; +INSERT INTO t (`id`,`a`) VALUES +(1,1), +(2,9999), +(3,10000); +connect deleter,localhost,root,,; +connect holder,localhost,root,,; +connect waiter,localhost,root,,; +connection deleter; +SET DEBUG_SYNC = +'lock_sec_rec_read_check_and_lock_has_locked + SIGNAL deleter_has_locked + WAIT_FOR waiter_has_locked'; +DELETE FROM t WHERE a = 9999; +connection holder; +SET DEBUG_SYNC= +'now WAIT_FOR deleter_has_locked'; +SET DEBUG_SYNC= +'lock_sec_rec_read_check_and_lock_has_locked SIGNAL holder_has_locked'; +DELETE FROM t WHERE a = 9999; +connection waiter; +SET DEBUG_SYNC= +'now WAIT_FOR holder_has_locked'; +SET DEBUG_SYNC= +'lock_sec_rec_read_check_and_lock_has_locked SIGNAL waiter_has_locked'; +DELETE FROM t WHERE a = 9999; +connection deleter; +connection holder; +connection waiter; +connection default; +disconnect deleter; +disconnect holder; +disconnect waiter; +DROP TABLE `t`; +SET DEBUG_SYNC='reset'; +CREATE TABLE `t`( +`id` INT NOT NULL PRIMARY KEY +) ENGINE=InnoDB; +INSERT INTO t (`id`) VALUES (1), (2); +connect holder,localhost,root,,; +connect waiter,localhost,root,,; +connection holder; +BEGIN; +SELECT id FROM t WHERE id=1 FOR UPDATE; +id +1 +SELECT id FROM t WHERE id=2 FOR UPDATE; +id +2 +connection waiter; +SET DEBUG_SYNC= +'lock_wait_suspend_thread_enter SIGNAL waiter_will_wait'; +SELECT id FROM t WHERE id = 1 FOR UPDATE; +connection holder; +SET DEBUG_SYNC= +'now WAIT_FOR waiter_will_wait'; +SELECT * FROM t FOR UPDATE; +id +1 +2 +COMMIT; +connection waiter; +id +1 +connection default; +disconnect holder; +disconnect waiter; +DROP TABLE `t`; +SET DEBUG_SYNC='reset'; diff --git a/mysql-test/suite/innodb/t/deadlock_on_lock_upgrade.test b/mysql-test/suite/innodb/t/deadlock_on_lock_upgrade.test new file mode 100644 index 00000000000..7f3f34ce063 --- /dev/null +++ b/mysql-test/suite/innodb/t/deadlock_on_lock_upgrade.test @@ -0,0 +1,143 @@ +--echo # +--echo # Bug #23755664 DEADLOCK WITH 3 CONCURRENT DELETES BY UNIQUE KEY +--echo # + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/count_sessions.inc + +--connection default +# There are various scenarious in which a transaction already holds "half" +# of a record lock (for example, a lock on the record but not on the gap) +# and wishes to "upgrade it" to a full lock (i.e. on both gap and record). +# This is often a cause for a deadlock, if there is another transaction +# which is already waiting for the lock being blocked by us: +# 1. our granted lock for one half +# 2. her waiting lock for the same half +# 3. our waiting lock for the whole + +# +# SCENARIO 1 +# +# In this scenario, three different threads try to delete the same row, +# identified by a secondary index key. +# This kind of operation (besides LOCK_IX on a table) requires +# an LOCK_REC_NOT_GAP|LOCK_REC|LOCK_X lock on a secondary index +# 1. `deleter` is the first to get the required lock +# 2. `holder` enqueues a waiting lock +# 3. `waiter` enqueues right after `holder` +# 4. `deleter` commits, releasing the lock, and granting it to `holder` +# 5. `holder` now observes that the row was deleted, so it needs to +# "seal the gap", by obtaining a LOCK_X|LOCK_REC, but.. +# 6. this causes a deadlock between `holder` and `waiter` +# +# This scenario does not fail if MDEV-10962 is not fixed because of MDEV-30225 +# fix, as the 'holder' does not "seal the gap" after 'deleter' was committed, +# because it was initially sealed, as row_search_mvcc() requests next-key lock +# after MDEV-30225 fix in the case when it requested not-gap lock before the +# fix. +# +# But let the scenario be in the tests, because it can fail if MDEV-30225 +# related code is changed + +CREATE TABLE `t`( + `id` INT, + `a` INT DEFAULT NULL, + PRIMARY KEY(`id`), + UNIQUE KEY `u`(`a`) +) ENGINE=InnoDB; + +INSERT INTO t (`id`,`a`) VALUES + (1,1), + (2,9999), + (3,10000); + +--connect(deleter,localhost,root,,) +--connect(holder,localhost,root,,) +--connect(waiter,localhost,root,,) + + +--connection deleter + SET DEBUG_SYNC = + 'lock_sec_rec_read_check_and_lock_has_locked + SIGNAL deleter_has_locked + WAIT_FOR waiter_has_locked'; + --send DELETE FROM t WHERE a = 9999 + +--connection holder + SET DEBUG_SYNC= + 'now WAIT_FOR deleter_has_locked'; + SET DEBUG_SYNC= + 'lock_sec_rec_read_check_and_lock_has_locked SIGNAL holder_has_locked'; + --send DELETE FROM t WHERE a = 9999 + +--connection waiter + SET DEBUG_SYNC= + 'now WAIT_FOR holder_has_locked'; + SET DEBUG_SYNC= + 'lock_sec_rec_read_check_and_lock_has_locked SIGNAL waiter_has_locked'; + --send DELETE FROM t WHERE a = 9999 + +--connection deleter + --reap + +--connection holder + --reap + +--connection waiter + --reap + +--connection default + +--disconnect deleter +--disconnect holder +--disconnect waiter + +DROP TABLE `t`; +SET DEBUG_SYNC='reset'; + +# SCENARIO 2 +# +# Here, we form a situation in which con1 has LOCK_REC_NOT_GAP on rows 1 and 2 +# con2 waits for lock on row 1, and then con1 wants to upgrade the lock on row 1, +# which might cause a deadlock, unless con1 properly notices that even though the +# lock on row 1 can not be upgraded, a separate LOCK_GAP can be obtaied easily. + +CREATE TABLE `t`( + `id` INT NOT NULL PRIMARY KEY +) ENGINE=InnoDB; + +INSERT INTO t (`id`) VALUES (1), (2); + +--connect(holder,localhost,root,,) +--connect(waiter,localhost,root,,) + +--connection holder + BEGIN; + SELECT id FROM t WHERE id=1 FOR UPDATE; + SELECT id FROM t WHERE id=2 FOR UPDATE; + +--connection waiter + SET DEBUG_SYNC= + 'lock_wait_suspend_thread_enter SIGNAL waiter_will_wait'; + --send SELECT id FROM t WHERE id = 1 FOR UPDATE + +--connection holder + SET DEBUG_SYNC= + 'now WAIT_FOR waiter_will_wait'; + SELECT * FROM t FOR UPDATE; + COMMIT; + +--connection waiter + --reap + +--connection default + +--disconnect holder +--disconnect waiter + +DROP TABLE `t`; +SET DEBUG_SYNC='reset'; + +--source include/wait_until_count_sessions.inc diff --git a/storage/innobase/include/lock0types.h b/storage/innobase/include/lock0types.h index cb04afdf9db..861885fc7a2 100644 --- a/storage/innobase/include/lock0types.h +++ b/storage/innobase/include/lock0types.h @@ -175,6 +175,26 @@ operator<<(std::ostream& out, const lock_rec_t& lock) #endif /* @} */ +/** +Checks if the `mode` is LOCK_S or LOCK_X (possibly ORed with LOCK_WAIT or +LOCK_REC) which means the lock is a +Next Key Lock, a.k.a. LOCK_ORDINARY, as opposed to Predicate Lock, +GAP lock, Insert Intention or Record Lock. +@param mode A mode and flags, of a lock. +@return true if the only bits set in `mode` are LOCK_S or LOCK_X and optionally +LOCK_WAIT or LOCK_REC */ +static inline bool lock_mode_is_next_key_lock(ulint mode) +{ + static_assert(LOCK_ORDINARY == 0, "LOCK_ORDINARY must be 0 (no flags)"); + ut_ad((mode & LOCK_TABLE) == 0); + mode&= ~(LOCK_WAIT | LOCK_REC); + ut_ad((mode & LOCK_WAIT) == 0); + ut_ad((mode & LOCK_TYPE_MASK) == 0); + ut_ad(((mode & ~(LOCK_MODE_MASK)) == LOCK_ORDINARY) == + (mode == LOCK_S || mode == LOCK_X)); + return (mode & ~(LOCK_MODE_MASK)) == LOCK_ORDINARY; +} + /** Lock struct; protected by lock_sys.mutex */ struct ib_lock_t { @@ -231,6 +251,13 @@ struct ib_lock_t return(type_mode & LOCK_REC_NOT_GAP); } + /** @return true if the lock is a Next Key Lock */ + bool is_next_key_lock() const + { + return is_record_lock() + && lock_mode_is_next_key_lock(type_mode); + } + bool is_insert_intention() const { return(type_mode & LOCK_INSERT_INTENTION); diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 26388ad95e2..8e75955c8af 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1054,6 +1054,14 @@ lock_rec_has_expl( static_cast( precise_mode & LOCK_MODE_MASK)) && !lock_get_wait(lock) + /* If we unfold the following expression, we will see it's + true when: + (heap_no is supremum) + or + (the found lock is LOCK_ORDINARY) + or + (the requested and the found lock modes are equal to each + other and equal to LOCK_REC_GAP | LOCK_REC_NOT_GAP). */ && (!lock_rec_get_rec_not_gap(lock) || (precise_mode & LOCK_REC_NOT_GAP) || heap_no == PAGE_HEAP_NO_SUPREMUM) @@ -1900,6 +1908,46 @@ lock_rec_add_to_queue( type_mode, block, heap_no, index, trx, caller_owns_trx_mutex); } +/** A helper function for lock_rec_lock_slow(), which grants a Next Key Lock +(either LOCK_X or LOCK_S as specified by `mode`) on <`block`,`heap_no`> in the +`index` to the `trx`, assuming that it already has a granted `held_lock`, which +is at least as strong as mode|LOCK_REC_NOT_GAP. It does so by either reusing the +lock if it already covers the gap, or by ensuring a separate GAP Lock, which in +combination with Record Lock satisfies the request. +@param[in] held_lock a lock granted to `trx` which is at least as strong + as mode|LOCK_REC_NOT_GAP +@param[in] mode requested lock mode: LOCK_X or LOCK_S +@param[in] block buffer block containing the record to be locked +@param[in] heap_no heap number of the record to be locked +@param[in] index index of record to be locked +@param[in] trx the transaction requesting the Next Key Lock */ +static void lock_reuse_for_next_key_lock(const lock_t *held_lock, ulint mode, + const buf_block_t *block, + ulint heap_no, dict_index_t *index, + trx_t *trx) +{ + ut_ad(lock_mutex_own()); + ut_ad(trx_mutex_own(trx)); + ut_ad(mode == LOCK_S || mode == LOCK_X); + ut_ad(lock_mode_is_next_key_lock(mode)); + + if (!held_lock->is_record_not_gap()) + { + ut_ad(held_lock->is_next_key_lock()); + return; + } + + /* We have a Record Lock granted, so we only need a GAP Lock. We assume + that GAP Locks do not conflict with anything. Therefore a GAP Lock + could be granted to us right now if we've requested: */ + mode|= LOCK_GAP; + ut_ad(nullptr == lock_rec_other_has_conflicting(mode, block, heap_no, trx)); + + /* It might be the case we already have one, so we first check that. */ + if (lock_rec_has_expl(mode, block, heap_no, trx) == nullptr) + lock_rec_add_to_queue(LOCK_REC | mode, block, heap_no, index, trx, true); +} + /*********************************************************************//** Tries to lock the specified record in the mode requested. If not immediately possible, enqueues a waiting lock request. This is a low-level function @@ -1950,8 +1998,17 @@ lock_rec_lock( lock->type_mode != (ulint(mode) | LOCK_REC) || lock_rec_get_n_bits(lock) <= heap_no) { + + ulint checked_mode= (heap_no != PAGE_HEAP_NO_SUPREMUM && + lock_mode_is_next_key_lock(mode)) + ? mode | LOCK_REC_NOT_GAP + : mode; + + const lock_t *held_lock= + lock_rec_has_expl(checked_mode, block, heap_no, trx); + /* Do nothing if the trx already has a strong enough lock on rec */ - if (!lock_rec_has_expl(mode, block, heap_no, trx)) + if (!held_lock) { if ( #ifdef WITH_WSREP @@ -1978,6 +2035,16 @@ lock_rec_lock( err= DB_SUCCESS_LOCKED_REC; } } + /* If checked_mode == mode, trx already has a strong enough lock on rec */ + else if (checked_mode != mode) + { + /* As check_mode != mode, the mode is Next Key Lock, which can not be + emulated by implicit lock (which are LOCK_REC_NOT_GAP only). */ + ut_ad(!impl); + + lock_reuse_for_next_key_lock(held_lock, mode, block, heap_no, index, + trx); + } } else if (!impl) { @@ -5844,6 +5911,8 @@ lock_sec_rec_read_check_and_lock( ut_ad(lock_rec_queue_validate(FALSE, block, rec, index, offsets)); + DEBUG_SYNC_C("lock_sec_rec_read_check_and_lock_has_locked"); + return(err); } From 94a8921e9dfde960f0ca802f53c4e13f8d5a66bb Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 5 Jul 2023 19:28:27 +0700 Subject: [PATCH 03/43] MDEV-29284 ANALYZE doesn't work with pushed derived tables There was no actual execution of the SQL of a pushed derived table, which caused "r_rows" to be always displayed as 0 and "r_total_time_ms" to show inaccurate numbers. This commit makes a derived table SQL to be executed by the storage engine, so the server is able to calculate the number of rows returned and measure the execution time more accurately --- .../suite/federated/federatedx_create_handlers.result | 4 ++-- sql/derived_handler.cc | 7 ------- sql/sql_select.h | 2 -- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/federated/federatedx_create_handlers.result b/mysql-test/suite/federated/federatedx_create_handlers.result index 612b89fa04e..666b6863295 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.result +++ b/mysql-test/suite/federated/federatedx_create_handlers.result @@ -190,7 +190,7 @@ FROM federated.t3, (SELECT * FROM federated.t1 WHERE id > 3) t WHERE federated.t3.name=t.name; id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra 1 PRIMARY t3 ALL NULL NULL NULL NULL 7 7.00 100.00 100.00 -1 PRIMARY ref key0 key0 18 federated.t3.name 2 0.00 100.00 100.00 +1 PRIMARY ref key0 key0 18 federated.t3.name 2 1.00 100.00 100.00 2 PUSHED DERIVED NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL SELECT * FROM federated.t3, (SELECT t1.name FROM federated.t1 @@ -241,7 +241,7 @@ ANALYZE "ref": ["federated.t3.name"], "r_loops": 7, "rows": 2, - "r_rows": 0, + "r_rows": 0.1429, "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100, diff --git a/sql/derived_handler.cc b/sql/derived_handler.cc index adb04e08c63..cddd1200f5d 100644 --- a/sql/derived_handler.cc +++ b/sql/derived_handler.cc @@ -40,7 +40,6 @@ Pushdown_derived::Pushdown_derived(TABLE_LIST *tbl, derived_handler *h) : derived(tbl), handler(h) { - is_analyze= handler->thd->lex->analyze_stmt; } @@ -57,12 +56,6 @@ int Pushdown_derived::execute() if ((err= handler->init_scan())) goto error; - if (is_analyze) - { - handler->end_scan(); - DBUG_RETURN(0); - } - while (!(err= handler->next_row())) { if (unlikely(thd->check_killed())) diff --git a/sql/sql_select.h b/sql/sql_select.h index 9b8bc72bbf9..5c00a77638a 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2545,8 +2545,6 @@ class derived_handler; class Pushdown_derived: public Sql_alloc { -private: - bool is_analyze; public: TABLE_LIST *derived; derived_handler *handler; From 8fb863e6a4fd322d704088dc39a5849c49549292 Mon Sep 17 00:00:00 2001 From: Yury Chaikou Date: Fri, 7 Jul 2023 10:33:47 +0200 Subject: [PATCH 04/43] MDEV-24712 get_partition_set is never executed in ha_partition::multi_range_key_create_key due to bitwise & with 0 constant use == to compare enum (despite the name it is not a bit flag) --- sql/ha_partition.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index ca02bf16d5e..7dfbf0268b4 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -6265,7 +6265,7 @@ int ha_partition::multi_range_key_create_key(RANGE_SEQ_IF *seq, m_mrr_range_current->ptr= m_mrr_range_current->key_multi_range.ptr; m_mrr_range_current->key_multi_range.ptr= m_mrr_range_current; - if (start_key->key && (start_key->flag & HA_READ_KEY_EXACT)) + if (start_key->key && (start_key->flag == HA_READ_KEY_EXACT)) get_partition_set(table, table->record[0], active_index, start_key, &m_part_spec); else From 02cd3675c4d211118c06478c50a7a515251bc2fc Mon Sep 17 00:00:00 2001 From: Lawrin Novitsky Date: Mon, 22 May 2023 15:07:05 +0200 Subject: [PATCH 05/43] MDEV-31064 Changes in a SP are not immediately seen in I_S.parameters If procedure is changed in one connection, and other procedure has already called the initial version of the procedure, the query to INFORMATION_SCHEMA.PARAMETERS would use obsolete information from sp cache for that connection. That happens because cache invalidating method only increments cache version, and does not flush (all) the cache(s), and changing of a procedure only invalidates cache, and removes the procedure's cache entry from local thread cache only. The fix adds the check if sp info obtained from the cache for forming of results for the query to I_S, is not obsoleted, and does not use it, if it is. The test has been added to main.information_schema. It changes the SP in one connection, and ensures, that the change is seen in the query to the I_S.PARAMETERS in other connection, that already has called the procedure before the change. --- mysql-test/main/information_schema.result | 22 ++++++++++++++++++++++ mysql-test/main/information_schema.test | 22 ++++++++++++++++++++++ sql/sp.cc | 4 +++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/information_schema.result b/mysql-test/main/information_schema.result index 2160eee08a7..e17bb73d4e7 100644 --- a/mysql-test/main/information_schema.result +++ b/mysql-test/main/information_schema.result @@ -2401,3 +2401,25 @@ progress # # End of 10.3 tests # +# +# MDEV-MDEV-31064 Changes of the procedure are not immediatly seen in queries to I_S.parameter from other connections +# +CREATE PROCEDURE sp1(IN p1 INT, IN p2 INT) +BEGIN +END; +connect con2, localhost, root,,; +CALL sp1(10, 20); +connection default; +CREATE OR REPLACE PROCEDURE sp1(p1 INT) +BEGIN +END; +connection con2; +SELECT COUNT(*) FROM information_schema.parameters WHERE SPECIFIC_NAME = 'sp1'; +COUNT(*) +1 +disconnect con2; +connection default; +DROP PROCEDURE sp1; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/information_schema.test b/mysql-test/main/information_schema.test index b9ed6326350..7b5884ef6b5 100644 --- a/mysql-test/main/information_schema.test +++ b/mysql-test/main/information_schema.test @@ -2114,3 +2114,25 @@ select progress from information_schema.processlist limit 1; --echo # --echo # End of 10.3 tests --echo # + +--echo # +--echo # MDEV-MDEV-31064 Changes of the procedure are not immediatly seen in queries to I_S.parameter from other connections +--echo # +CREATE PROCEDURE sp1(IN p1 INT, IN p2 INT) +BEGIN +END; +--connect(con2, localhost, root,,) +CALL sp1(10, 20); +--connection default +CREATE OR REPLACE PROCEDURE sp1(p1 INT) +BEGIN +END; +--connection con2 +SELECT COUNT(*) FROM information_schema.parameters WHERE SPECIFIC_NAME = 'sp1'; +--disconnect con2 +--connection default +DROP PROCEDURE sp1; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/sql/sp.cc b/sql/sp.cc index af9c7901c5a..87af91187e7 100644 --- a/sql/sp.cc +++ b/sql/sp.cc @@ -3018,7 +3018,9 @@ Sp_handler::sp_load_for_information_schema(THD *thd, TABLE *proc_table, sp_cache **spc= get_cache(thd); sp_name sp_name_obj(&db, &name, true); // This can change "name" *free_sp_head= 0; - if ((sp= sp_cache_lookup(spc, &sp_name_obj))) + sp= sp_cache_lookup(spc, &sp_name_obj); + + if (sp && !(sp->sp_cache_version() < sp_cache_version())) { return sp; } From 12a5fb4b36b573764564eab05cb2575c5a59b471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 10 Jul 2023 13:46:34 +0300 Subject: [PATCH 06/43] MDEV-31641 innochecksum dies with Floating point exception print_summary(): Skip index_ids for which index.pages is 0. Tablespaces may contain some freed pages that used to refer to indexes or tables that were dropped. --- extra/innochecksum.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 67528827dc0..8adc9ddf05c 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -1192,11 +1192,15 @@ print_summary( fprintf(fil_out, "index_id\t#pages\t\t#leaf_pages\t#recs_per_page" "\t#bytes_per_page\n"); - for (std::map::const_iterator it = index_ids.begin(); - it != index_ids.end(); it++) { - const per_index_stats& index = it->second; + for (const auto &ids : index_ids) { + const per_index_stats& index = ids.second; + if (!index.pages) { + DBUG_ASSERT(index.free_pages); + continue; + } + fprintf(fil_out, "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", - it->first, index.pages, index.leaf_pages, + ids.first, index.pages, index.leaf_pages, index.total_n_recs / index.pages, index.total_data_bytes / index.pages); } From 7a5c984fa35fff3f86b553b9dd5171cfb511dae8 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 10 Jul 2023 18:43:56 +0300 Subject: [PATCH 07/43] MDEV-20010 Equal on two RANK window functions create wrong result The problematic query outlined a bug in window functions sorting optimization. When multiple window functions are present in a query, we sort the sorting key (as defined by PARTITION BY and ORDER BY) from generic to specific. SELECT RANK() OVER (ORDER BY const_col) as r1, RANK() OVER (ORDER BY const_col, a) as r2, RANK() OVER (PARTITION BY c) as r3, RANK() OVER (PARTITION BY c ORDER BY b) as r4 FROM table; For these functions, the sorting we need to do for window function computations are: [(const_col), (const_col, a)] and [(c), (c, b)]. Instead of doing 4 different sort order, the sorts grouped within [] are compatible and we can use the most *specific* sort to cover both window functions. The bug was caused by an incorrect flagging of which sort is most specific for a compatible group of functions. In our specific test case, instead of picking (const_col, a) as the most specific sort, it would only sort by (const_col), which lead to wrong results for rank function. By ensuring that we pick the last sort key before an "incompatible sort" flag is met in our "ordered array of sorting specifications", we guarantee correct results. --- mysql-test/main/cte_recursive.result | 48 ++++++++++++++++++++++++++++ mysql-test/main/cte_recursive.test | 34 ++++++++++++++++++++ sql/sql_window.cc | 2 +- 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/cte_recursive.result b/mysql-test/main/cte_recursive.result index 0b7b216f1fe..63fcb63c7ee 100644 --- a/mysql-test/main/cte_recursive.result +++ b/mysql-test/main/cte_recursive.result @@ -5800,4 +5800,52 @@ a a 9 9 10 10 drop table t1; +# +# MDEV-20010 Equal on two RANK window functions create wrong result +# +create table t1 (a int, b int) engine= innodb; +insert into t1 values (4, -2), (3, -1); +SELECT RANK() OVER (ORDER BY D.C) = RANK() OVER (ORDER BY B.a) FROM +(SELECT 5 AS C FROM t1) as D, (SELECT t1.b AS A FROM t1) AS B; +RANK() OVER (ORDER BY D.C) = RANK() OVER (ORDER BY B.a) +1 +1 +0 +0 +select b, rank() over (order by c) , rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; +b rank() over (order by c) rank() over (order by dt1.b) +-2 1 1 +-2 1 1 +-1 1 3 +-1 1 3 +select b, rank() over (order by c) , rank() over (order by dt1.b), +rank() over (order by c) = rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; +b rank() over (order by c) rank() over (order by dt1.b) rank() over (order by c) = rank() over (order by dt1.b) +-2 1 1 1 +-2 1 1 1 +-1 1 3 0 +-1 1 3 0 +alter table t1 engine=myisam; +select b, rank() over (order by c) , rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; +b rank() over (order by c) rank() over (order by dt1.b) +-2 1 1 +-2 1 1 +-1 1 3 +-1 1 3 +create view v1 as select b,5 as c from t1; +select b, rank() over (order by c) from v1 order by b; +b rank() over (order by c) +-2 1 +-1 1 +drop view v1; +drop table t1; # End of 10.4 tests diff --git a/mysql-test/main/cte_recursive.test b/mysql-test/main/cte_recursive.test index 4d21a494bdc..6053025499a 100644 --- a/mysql-test/main/cte_recursive.test +++ b/mysql-test/main/cte_recursive.test @@ -1,4 +1,5 @@ --source include/default_optimizer_switch.inc +--source include/have_innodb.inc create table t1 (a int, b varchar(32)); insert into t1 values @@ -4013,4 +4014,37 @@ with cte_e as ( drop table t1; +--echo # +--echo # MDEV-20010 Equal on two RANK window functions create wrong result +--echo # + +create table t1 (a int, b int) engine= innodb; +insert into t1 values (4, -2), (3, -1); + +SELECT RANK() OVER (ORDER BY D.C) = RANK() OVER (ORDER BY B.a) FROM +(SELECT 5 AS C FROM t1) as D, (SELECT t1.b AS A FROM t1) AS B; + +select b, rank() over (order by c) , rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; + +select b, rank() over (order by c) , rank() over (order by dt1.b), +rank() over (order by c) = rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; + +alter table t1 engine=myisam; +select b, rank() over (order by c) , rank() over (order by dt1.b) +from +(select 5 as c from t1) as dt, +(select b from t1) as dt1; + +create view v1 as select b,5 as c from t1; +select b, rank() over (order by c) from v1 order by b; + +drop view v1; +drop table t1; + --echo # End of 10.4 tests diff --git a/sql/sql_window.cc b/sql/sql_window.cc index ff7554b0cd7..df43efe18a4 100644 --- a/sql/sql_window.cc +++ b/sql/sql_window.cc @@ -3076,7 +3076,7 @@ bool Window_funcs_sort::setup(THD *thd, SQL_SELECT *sel, spec= win_func->window_spec; int win_func_order_elements= spec->partition_list->elements + spec->order_list->elements; - if (win_func_order_elements > longest_order_elements) + if (win_func_order_elements >= longest_order_elements) { win_func_with_longest_order= win_func; longest_order_elements= win_func_order_elements; From 23d53913fb61fc4f19ff800577e11e199d546347 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 15 Jun 2023 15:18:40 +1000 Subject: [PATCH 08/43] MDEV-27038 Custom configuration file procedure does not work with Docker Desktop for Windows 10+ Docker when mounting a configuration file into a Windows exposes the file with permission 0777. These world writable files are ignored by by MariaDB. Add the access check such that filesystem RO or immutable file is counted as sufficient protection on the file. Test: $ mkdir /tmp/src $ vi /tmp/src/my.cnf $ chmod 666 /tmp/src/my.cnf $ mkdir /tmp/dst $ sudo mount --bind /tmp/src /tmp/dst -o ro $ ls -la /tmp/dst total 4 drwxr-xr-x. 2 dan dan 60 Jun 15 15:12 . drwxrwxrwt. 25 root root 660 Jun 15 15:13 .. -rw-rw-rw-. 1 dan dan 10 Jun 15 15:12 my.cnf $ mount | grep dst tmpfs on /tmp/dst type tmpfs (ro,seclabel,nr_inodes=1048576,inode64) strace client/mariadb --defaults-file=/tmp/dst/my.cnf newfstatat(AT_FDCWD, "/tmp/dst/my.cnf", {st_mode=S_IFREG|0666, st_size=10, ...}, 0) = 0 access("/tmp/dst/my.cnf", W_OK) = -1 EROFS (Read-only file system) openat(AT_FDCWD, "/tmp/dst/my.cnf", O_RDONLY|O_CLOEXEC) = 3 The one failing test, but this isn't a regression, just not a total fix: $ chmod u-w /tmp/src/my.cnf $ ls -la /tmp/src/my.cnf -r--rw-rw-. 1 dan dan 18 Jun 16 10:22 /tmp/src/my.cnf $ strace -fe trace=access client/mariadb --defaults-file=/tmp/dst/my.cnf access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) access("/etc/system-fips", F_OK) = -1 ENOENT (No such file or directory) access("/tmp/dst/my.cnf", W_OK) = -1 EACCES (Permission denied) Warning: World-writable config file '/tmp/dst/my.cnf' is ignored Windows test (Docker Desktop ~4.21) which was the important one to fix: dan@LAPTOP-5B5P7RCK:~$ docker run --rm -v /mnt/c/Users/danie/Desktop/conf:/etc/mysql/conf.d/:ro -e MARIADB_ROOT_PASSWORD=bob quay.io/m ariadb-foundation/mariadb-devel:10.4-MDEV-27038-ro-mounts-pkgtest ls -la /etc/mysql/conf.d total 4 drwxrwxrwx 1 root root 512 Jun 15 13:57 . drwxr-xr-x 4 root root 4096 Jun 15 07:32 .. -rwxrwxrwx 1 root root 43 Jun 15 13:56 myapp.cnf root@a59b38b45af1:/# strace -fe trace=access mariadb access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) access("/etc/mysql/conf.d/myapp.cnf", W_OK) = -1 EROFS (Read-only file system) --- mysys/my_default.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/mysys/my_default.c b/mysys/my_default.c index 7c6ef8fffa0..1aaf75b88b1 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -786,12 +786,27 @@ static int search_default_file_with_ext(Process_option_func opt_handler, if (!my_stat(name,&stat_info,MYF(0))) return 1; /* - Ignore world-writable regular files. - This is mainly done to protect us to not read a file created by - the mysqld server, but the check is still valid in most context. + Ignore world-writable regular files (exceptions apply). + This is mainly done to protect us to not read a file that may be + modified by anyone. + + Also check access so that read only mounted (EROFS) + or immutable files (EPERM) that are suitable protections. + + The main case we are allowing is a container readonly volume mount + from a filesystem that doesn't have unix permissions. This will + have a 0777 permission and access will set errno = EROFS. + + Note if a ROFS has a file with permissions 04n6, access sets errno + EACCESS, rather the ROFS, so in this case we'll error, even though + the ROFS is protecting the file. + + An ideal, race free, implementation would do fstat / fstatvfs / ioctl + for permission, read only filesystem, and immutability resprectively. */ if ((stat_info.st_mode & S_IWOTH) && - (stat_info.st_mode & S_IFMT) == S_IFREG) + (stat_info.st_mode & S_IFMT) == S_IFREG && + (access(name, W_OK) == 0 || (errno != EROFS && errno != EPERM))) { fprintf(stderr, "Warning: World-writable config file '%s' is ignored\n", name); From b4646c675c09b9152db32bef6b88c27121e40d2e Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 16 Jun 2023 15:33:51 +0200 Subject: [PATCH 09/43] Misc. small cleanups unrelated to any particular MDEV Signed-off-by: Kristian Nielsen --- sql/log.cc | 4 ++-- sql/sql_class.cc | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 67e2d2f5b2a..06ba27c4b79 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -7587,7 +7587,7 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) Setting this flag may or may not be seen by the other thread, but we are safe in any case: The other thread will set queued_by_other under - its LOCK_wait_commit, and we will not check queued_by_other only after + its LOCK_wait_commit, and we will not check queued_by_other until after we have been woken up. */ wfc->opaque_pointer= orig_entry; @@ -7684,7 +7684,7 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) is pointed to by `last` (we do not use NULL to terminate the list). As we process an entry, any waiters for that entry are added at the end of - the list, to be processed in subsequent iterations. The the entry is added + the list, to be processed in subsequent iterations. Then the entry is added to the group_commit_queue. This continues until the list is exhausted, with all entries ever added eventually processed. diff --git a/sql/sql_class.cc b/sql/sql_class.cc index dc58d2a250a..a0d05846ca8 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7643,7 +7643,7 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) { PSI_stage_info old_stage; wait_for_commit *loc_waitee; - bool backup_lock_released= 0; + bool backup_lock_released= false; /* Release MDL_BACKUP_COMMIT LOCK while waiting for other threads to commit @@ -7653,7 +7653,7 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) */ if (thd->backup_commit_lock && thd->backup_commit_lock->ticket) { - backup_lock_released= 1; + backup_lock_released= true; thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); thd->backup_commit_lock->ticket= 0; } @@ -7706,14 +7706,14 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) use within enter_cond/exit_cond. */ DEBUG_SYNC(thd, "wait_for_prior_commit_killed"); - if (backup_lock_released) + if (unlikely(backup_lock_released)) thd->mdl_context.acquire_lock(thd->backup_commit_lock, thd->variables.lock_wait_timeout); return wakeup_error; end: thd->EXIT_COND(&old_stage); - if (backup_lock_released) + if (unlikely(backup_lock_released)) thd->mdl_context.acquire_lock(thd->backup_commit_lock, thd->variables.lock_wait_timeout); return wakeup_error; From 60bec1d54d320e86f2c09407e1981c901a675c1c Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sat, 10 Jun 2023 22:36:16 +0200 Subject: [PATCH 10/43] MDEV-13915: STOP SLAVE takes very long time on a busy system At STOP SLAVE, worker threads will continue applying event groups until the end of the current GCO before stopping. This is a left-over from when only conservative mode was available. In optimistic and aggressive mode, often _all_ queued event will be in the same GCO, and slave stop will be needlessly delayed. This patch instead records at STOP SLAVE time the latest (highest sub_id) event group that has started. Then worker threads will continue to apply event groups up to that event group, but skip any following. The result is that each worker thread will complete its currently running event group, and then the slave will stop. If the slave is caught up, and STOP SLAVE is run in the middle of an event group that is already executing in a worker thread, then that event group will be rolled back and the slave stop immediately, as normal. Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- sql/rpl_parallel.cc | 27 +++++++++++++-------------- sql/rpl_parallel.h | 10 +++++----- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 6ca582e4f21..98256118909 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -395,12 +395,13 @@ do_gco_wait(rpl_group_info *rgi, group_commit_orderer *gco, } while (wait_count > entry->count_committing_event_groups); } - if (entry->force_abort && wait_count > entry->stop_count) + if (entry->force_abort && rgi->gtid_sub_id > entry->stop_sub_id) { /* - We are stopping (STOP SLAVE), and this event group is beyond the point - where we can safely stop. So return a flag that will cause us to skip, - rather than execute, the following events. + We are stopping (STOP SLAVE), and this event group need not be applied + before we can safely stop. So return a flag that will cause us to skip, + rather than execute, the following events. Once all queued events have + been skipped, the STOP SLAVE is complete (for this thread). */ return true; } @@ -2357,20 +2358,18 @@ rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) are also executed, so that we stop at a consistent point in the binlog stream (per replication domain). - All event groups wait for e->count_committing_event_groups to reach - the value of group_commit_orderer::wait_count before starting to - execute. Thus, at this point we know that any event group with a - strictly larger wait_count are safe to skip, none of them can have - started executing yet. So we set e->stop_count here and use it to - decide in the worker threads whether to continue executing an event - group or whether to skip it, when force_abort is set. + At this point, we are holding LOCK_parallel_entry, and we know that no + event group after e->largest_started_sub_id has started running yet. We + record this value in e->stop_sub_id, and then each event group can check + their own sub_id against it. If their sub_id is strictly larger, then + that event group will be skipped. If we stop due to reaching the START SLAVE UNTIL condition, then we need to continue executing any queued events up to that point. */ e->force_abort= true; - e->stop_count= rli->stop_for_until ? - e->count_queued_event_groups : e->count_committing_event_groups; + e->stop_sub_id= rli->stop_for_until ? + e->current_sub_id : e->largest_started_sub_id; mysql_mutex_unlock(&e->LOCK_parallel_entry); for (j= 0; j < e->rpl_thread_max; ++j) { @@ -2426,7 +2425,7 @@ rpl_parallel::stop_during_until() e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); mysql_mutex_lock(&e->LOCK_parallel_entry); if (e->force_abort) - e->stop_count= e->count_committing_event_groups; + e->stop_sub_id= e->largest_started_sub_id; mysql_mutex_unlock(&e->LOCK_parallel_entry); } } diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index 650aa06e504..b7304d204ee 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -276,13 +276,13 @@ struct rpl_parallel_entry { /* At STOP SLAVE (force_abort=true), we do not want to process all events in the queue (which could unnecessarily delay stop, if a lot of events happen - to be queued). The stop_count provides a safe point at which to stop, so + to be queued). The stop_sub_id provides a safe point at which to stop, so that everything before becomes committed and nothing after does. The value - corresponds to group_commit_orderer::wait_count; if wait_count is less than - or equal to stop_count, we execute the associated event group, else we - skip it (and all following) and stop. + corresponds to rpl_group_info::gtid_sub_id; if that is less than or equal + to stop_sub_id, we execute the associated event group, else we skip it (and + all following) and stop. */ - uint64 stop_count; + uint64 stop_sub_id; /* Cyclic array recording the last rpl_thread_max worker threads that we From a8ea6627a4d3a099e69a88962b71649da8dadfa4 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 11 Jun 2023 17:44:58 +0200 Subject: [PATCH 11/43] MDEV-31448: Killing a replica thread awaiting its GCO can hang/crash a parallel replica The problem was an incorrect unmark_start_commit() in signal_error_to_sql_driver_thread(). If an event group gets an error, this unmark could run after the following GCO started, and the subsequent re-marking could access de-allocated GCO. The offending unmark_start_commit() looks obviously incorrect, and the fix is to just remove it. It was introduced in the MDEV-8302 patch, the commit message of which suggests it was added there solely to satisfy an assertion in ha_rollback_trans(). So update this assertion instead to not trigger for event groups that experienced an error (rgi->worker_error). When an error occurs in an event group, all following event groups are skipped anyway, so the unmark should never be needed in this case. Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- sql/handler.cc | 13 +++++++++++-- sql/rpl_parallel.cc | 5 ----- sql/rpl_parallel.h | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index f7d781eb82f..48ce7b3f1f8 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1898,13 +1898,22 @@ int ha_rollback_trans(THD *thd, bool all) attempt. Otherwise those following transactions can run too early, and possibly cause replication to fail. See comments in retry_event_group(). + (This concerns rollbacks due to temporary errors where the transaction + will be retried afterwards. For non-recoverable errors, following + transactions will not start but just be skipped as the worker threads + perform the error stop). + There were several bugs with this in the past that were very hard to track down (MDEV-7458, MDEV-8302). So we add here an assertion for rollback without signalling following transactions. And in release builds, we explicitly do the signalling before rolling back. */ - DBUG_ASSERT(!(thd->rgi_slave && thd->rgi_slave->did_mark_start_commit)); - if (thd->rgi_slave && thd->rgi_slave->did_mark_start_commit) + DBUG_ASSERT( !(thd->rgi_slave && + !thd->rgi_slave->worker_error && + thd->rgi_slave->did_mark_start_commit)); + if (thd->rgi_slave && + !thd->rgi_slave->worker_error && + thd->rgi_slave->did_mark_start_commit) thd->rgi_slave->unmark_start_commit(); } #endif diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 98256118909..1dab1224286 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -286,16 +286,11 @@ static void signal_error_to_sql_driver_thread(THD *thd, rpl_group_info *rgi, int err) { rgi->worker_error= err; - /* - In case we get an error during commit, inform following transactions that - we aborted our commit. - */ DBUG_EXECUTE_IF("hold_worker2_favor_worker3", { if (rgi->current_gtid.seq_no == 2002) { debug_sync_set_action(thd, STRING_WITH_LEN("now WAIT_FOR cont_worker2")); }}); - rgi->unmark_start_commit(); rgi->cleanup_context(thd, true); rgi->rli->abort_slave= true; rgi->rli->stop_for_until= false; diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index b7304d204ee..6b03306692b 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -91,6 +91,10 @@ struct group_commit_orderer { }; uint8 flags; #ifndef DBUG_OFF + /* + Flag set when the GCO has been freed and entered the free list, to catch + (in debug) errors in the complex lifetime of this object. + */ bool gc_done; #endif }; From 5d61442c85b35ce382ac2afeaad5c19e630b41dd Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 15 Jun 2023 21:35:53 +0200 Subject: [PATCH 12/43] MDEV-31448: Killing a replica thread awaiting its GCO can hang/crash a parallel replica The problem is that when a worker thread is (user) killed in wait_for_prior_commit, the event group may complete out-of-order since the wait for prior commit was aborted by the kill. This fix ensures that event groups will always complete in-order, even in the error case. This is done in finish_event_group() by doing an extra wait_for_prior_commit(), if necessary, that ignores kills. This fix supersedes the fix for MDEV-30780, so the earlier fix for that is reverted in this patch. Also fix that an error from wait_for_prior_commit() inside finish_event_group() would not signal the error to wakeup_subsequent_commits(). Based on earlier work by Brandon Nesterenko and Andrei Elkin, with some changes to simplify the semantics of wait_for_prior_commit() and make the code more robust to future changes. Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- sql/rpl_parallel.cc | 59 ++++++++++++++++++++++++++++++++++++--------- sql/sql_class.cc | 11 +++++++-- sql/sql_class.h | 10 ++++---- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 1dab1224286..ed8858d67ba 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -27,6 +27,9 @@ struct rpl_parallel_thread_pool global_rpl_thread_pool; static void signal_error_to_sql_driver_thread(THD *thd, rpl_group_info *rgi, int err); +static void +register_wait_for_prior_event_group_commit(rpl_group_info *rgi, + rpl_parallel_entry *entry); static int rpt_handle_event(rpl_parallel_thread::queued_event *qev, @@ -151,15 +154,35 @@ finish_event_group(rpl_parallel_thread *rpt, uint64 sub_id, int err; thd->get_stmt_da()->set_overwrite_status(true); + + if (unlikely(rgi->worker_error)) + { + /* + In case a previous wait was killed, we need to re-register to be able to + repeat the wait. + + And before doing that, we un-register any previous registration (in case + we got an error earlier and skipped waiting). + */ + thd->wait_for_commit_ptr->unregister_wait_for_prior_commit(); + mysql_mutex_lock(&entry->LOCK_parallel_entry); + register_wait_for_prior_event_group_commit(rgi, entry); + mysql_mutex_unlock(&entry->LOCK_parallel_entry); + } + /* Remove any left-over registration to wait for a prior commit to complete. Normally, such wait would already have been removed at this point by wait_for_prior_commit() called from within COMMIT - processing. However, in case of MyISAM and no binlog, we might not - have any commit processing, and so we need to do the wait here, - before waking up any subsequent commits, to preserve correct - order of event execution. Also, in the error case we might have - skipped waiting and thus need to remove it explicitly. + processing. + + However, in case of MyISAM and no binlog, we might not have any commit + processing, and so we need to do the wait here, before waking up any + subsequent commits, to preserve correct order of event execution. + + Also, in the error case we might have skipped waiting and thus need to + remove it explicitly. Or the wait might have been killed and we need to + repeat the registration and the wait. It is important in the non-error case to do a wait, not just an unregister. Because we might be last in a group-commit that is @@ -172,8 +195,18 @@ finish_event_group(rpl_parallel_thread *rpt, uint64 sub_id, all earlier event groups have also committed; this way no more mark_start_commit() calls can be made and it is safe to de-allocate the GCO. + + Thus this final wait is done with kill ignored during the wait. This is + fine, at this point there is no active query or transaction to abort, and + the thread will continue as soon as earlier event groups complete. + + Note though, that in the non-error case there is no guarantee that + finish_event_group() will be run in-order. For example, a successful + binlog group commit will wakeup all participating event groups + simultaneously so only thread scheduling will decide the order in which + finish_event_group() calls acquire LOCK_parallel_entry. */ - err= wfc->wait_for_prior_commit(thd); + err= wfc->wait_for_prior_commit(thd, false); if (unlikely(err) && !rgi->worker_error) signal_error_to_sql_driver_thread(thd, rgi, err); thd->wait_for_commit_ptr= NULL; @@ -242,8 +275,7 @@ finish_event_group(rpl_parallel_thread *rpt, uint64 sub_id, not yet started should just skip their group, preparing for stop of the SQL driver thread. */ - if (unlikely(rgi->worker_error) && - entry->stop_on_error_sub_id == (uint64)ULONGLONG_MAX) + if (unlikely(rgi->worker_error) && entry->stop_on_error_sub_id > sub_id) entry->stop_on_error_sub_id= sub_id; mysql_mutex_unlock(&entry->LOCK_parallel_entry); #ifdef ENABLED_DEBUG_SYNC @@ -820,12 +852,15 @@ do_retry: for (;;) { mysql_mutex_lock(&entry->LOCK_parallel_entry); - register_wait_for_prior_event_group_commit(rgi, entry); - if (!(entry->stop_on_error_sub_id == (uint64) ULONGLONG_MAX || + if (rgi->gtid_sub_id < entry->stop_on_error_sub_id #ifndef DBUG_OFF - (DBUG_EVALUATE_IF("simulate_mdev_12746", 1, 0)) || + || DBUG_EVALUATE_IF("simulate_mdev_12746", 1, 0) #endif - rgi->gtid_sub_id < entry->stop_on_error_sub_id)) + ) + { + register_wait_for_prior_event_group_commit(rgi, entry); + } + else { /* A failure of a preceeding "parent" transaction may not be diff --git a/sql/sql_class.cc b/sql/sql_class.cc index a0d05846ca8..d2a3cbcf613 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7635,11 +7635,18 @@ wait_for_commit::register_wait_for_prior_commit(wait_for_commit *waitee) with register_wait_for_prior_commit(). If the commit already completed, returns immediately. + If ALLOW_KILL is set to true (the default), the wait can be aborted by a + kill. In case of kill, the wait registration is still removed, so another + call of unregister_wait_for_prior_commit() is needed to later retry the + wait. If ALLOW_KILL is set to false, then kill will be ignored and this + function will not return until the prior commit (if any) has called + wakeup_subsequent_commits(). + If thd->backup_commit_lock is set, release it while waiting for other threads */ int -wait_for_commit::wait_for_prior_commit2(THD *thd) +wait_for_commit::wait_for_prior_commit2(THD *thd, bool allow_kill) { PSI_stage_info old_stage; wait_for_commit *loc_waitee; @@ -7664,7 +7671,7 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) &stage_waiting_for_prior_transaction_to_commit, &old_stage); while ((loc_waitee= this->waitee.load(std::memory_order_relaxed)) && - likely(!thd->check_killed(1))) + (!allow_kill || likely(!thd->check_killed(1)))) mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); if (!loc_waitee) { diff --git a/sql/sql_class.h b/sql/sql_class.h index 4e5aba33443..4487a67c76d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2144,14 +2144,14 @@ struct wait_for_commit bool commit_started; void register_wait_for_prior_commit(wait_for_commit *waitee); - int wait_for_prior_commit(THD *thd) + int wait_for_prior_commit(THD *thd, bool allow_kill=true) { /* Quick inline check, to avoid function call and locking in the common case where no wakeup is registered, or a registered wait was already signalled. */ if (waitee.load(std::memory_order_acquire)) - return wait_for_prior_commit2(thd); + return wait_for_prior_commit2(thd, allow_kill); else { if (wakeup_error) @@ -2205,7 +2205,7 @@ struct wait_for_commit void wakeup(int wakeup_error); - int wait_for_prior_commit2(THD *thd); + int wait_for_prior_commit2(THD *thd, bool allow_kill); void wakeup_subsequent_commits2(int wakeup_error); void unregister_wait_for_prior_commit2(); @@ -4726,10 +4726,10 @@ public: } wait_for_commit *wait_for_commit_ptr; - int wait_for_prior_commit() + int wait_for_prior_commit(bool allow_kill=true) { if (wait_for_commit_ptr) - return wait_for_commit_ptr->wait_for_prior_commit(this); + return wait_for_commit_ptr->wait_for_prior_commit(this, allow_kill); return 0; } void wakeup_subsequent_commits(int wakeup_error) From d4309d4830c8889286ab586c04dc5545ba9bf0a4 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 15 Jun 2023 21:46:01 +0200 Subject: [PATCH 13/43] MDEV-31448: Killing a replica thread awaiting its GCO can hang/crash a parallel replica Various test cases for the bugs around MDEV-31448. Test cases due to Brandon Nesterenko, thanks! Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- .../rpl/include/mdev-31448_conservative.inc | 68 +++++++++ .../rpl/include/mdev-31448_optimistic.inc | 94 ++++++++++++ ...ev-31448_kill_ooo_finish_optimistic.result | 52 +++++++ .../suite/rpl/r/rpl_parallel_kill.result | 142 ++++++++++++++++++ ...mdev-31448_kill_ooo_finish_optimistic.test | 93 ++++++++++++ mysql-test/suite/rpl/t/rpl_parallel_kill.test | 15 ++ 6 files changed, 464 insertions(+) create mode 100644 mysql-test/suite/rpl/include/mdev-31448_conservative.inc create mode 100644 mysql-test/suite/rpl/include/mdev-31448_optimistic.inc create mode 100644 mysql-test/suite/rpl/r/mdev-31448_kill_ooo_finish_optimistic.result create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_kill.result create mode 100644 mysql-test/suite/rpl/t/mdev-31448_kill_ooo_finish_optimistic.test create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_kill.test diff --git a/mysql-test/suite/rpl/include/mdev-31448_conservative.inc b/mysql-test/suite/rpl/include/mdev-31448_conservative.inc new file mode 100644 index 00000000000..9a2884439f6 --- /dev/null +++ b/mysql-test/suite/rpl/include/mdev-31448_conservative.inc @@ -0,0 +1,68 @@ +--connection master +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; +insert into t1 values (1); +--source include/save_master_gtid.inc + +--connection slave +call mtr.add_suppression("Slave: Commit failed due to failure of an earlier commit on which this one depends"); + +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +set @save.slave_parallel_threads= @@global.slave_parallel_threads; +set @save.slave_parallel_mode= @@global.slave_parallel_mode; +set @@global.slave_parallel_threads= 3; +set @@global.slave_parallel_mode= CONSERVATIVE; +--connection slave1 +BEGIN; +update t1 set a=2 where a=1; + +--connection master +SET @old_dbug= @@SESSION.debug_dbug; +SET @@SESSION.debug_dbug="+d,binlog_force_commit_id"; + +# GCO 1 +SET @commit_id= 10000; +# T1 +update t1 set a=2 where a=1; +# T2 +insert into t2 values (1); + +# GCO 2 +SET @commit_id= 10001; +# T3 +insert into t1 values (3); + +--connection slave +--source include/start_slave.inc + +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Update_rows_log_event::find_row(-1)' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc + +--let $t3_tid= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE 'Waiting for prior transaction to start commit%'` +--evalp kill $t3_tid + +--connection slave1 +commit; + +--connection slave +--let $slave_timeout=1032 +--source include/wait_for_slave_sql_to_stop.inc + +update t1 set a=1 where a=2; +set @@global.slave_parallel_threads = @save.slave_parallel_threads; +set @@global.slave_parallel_mode = @save.slave_parallel_mode; +--source include/start_slave.inc + +--echo # +--echo # Cleanup +--connection master +DROP TABLE t1, t2; +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc diff --git a/mysql-test/suite/rpl/include/mdev-31448_optimistic.inc b/mysql-test/suite/rpl/include/mdev-31448_optimistic.inc new file mode 100644 index 00000000000..9b72181d249 --- /dev/null +++ b/mysql-test/suite/rpl/include/mdev-31448_optimistic.inc @@ -0,0 +1,94 @@ +--echo # MDEV-31448 OOO finish event group by killed worker +# The test demonstrates how a killed worker access gco lists +# in finish_event_group() out-of-order to fire +# DBUG_ASSERT(!tmp_gco->next_gco || tmp_gco->last_sub_id > sub_id); +# in the buggy version. + +--echo # Initialize test data +--connection master +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; + +insert into t1 values (1); +--source include/save_master_gtid.inc + +--connection slave +call mtr.add_suppression("Connection was killed"); +call mtr.add_suppression("Can.t find record"); + +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +set @save.slave_parallel_threads= @@global.slave_parallel_threads; +set @save.slave_parallel_mode= @@global.slave_parallel_mode; +set @@global.slave_parallel_threads= 3; +set @@global.slave_parallel_mode= OPTIMISTIC; + +--connection slave1 +begin; +update t1 set a=2 where a=1; + +--connection master +set @old_dbug= @@session.debug_dbug; +set @@session.debug_dbug="+d,binlog_force_commit_id"; + +# GCO 1 +set @commit_id= 10000; +# T1 +update t1 set a=2 where a=1; + +if (!$killed_trx_commits) +{ +set @commit_id= 10001; +# T2 +set statement skip_parallel_replication=1 for insert into t2 values (1); +} + +if ($killed_trx_commits) +{ +insert into t2 values (1); +} +# GCO 2 +# T3 +drop table t2; + +--connection slave +--source include/start_slave.inc + +--echo # wait for T1 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Update_rows_log_event::find_row(-1)' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc + +--echo # wait for T2 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc +--let $t2_tid= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE 'Waiting for prior transaction to commit%' and command LIKE 'Slave_worker'` +--echo # wait for T3 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc + +--evalp kill $t2_tid +# give some little time for T2 to re-sink into the same state +--let $slave_param=Last_Errno +--let $slave_param_value=1927 +--source include/wait_for_slave_param.inc +--connection slave1 +commit; + +--connection slave +--let $slave_timeout=1032 +--source include/wait_for_slave_sql_to_stop.inc + +update t1 set a=1 where a=2; +set @@global.slave_parallel_threads = @save.slave_parallel_threads; +set @@global.slave_parallel_mode = @save.slave_parallel_mode; +--source include/start_slave.inc + +--echo # +--echo # Cleanup +--connection master +drop table t1; +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc + diff --git a/mysql-test/suite/rpl/r/mdev-31448_kill_ooo_finish_optimistic.result b/mysql-test/suite/rpl/r/mdev-31448_kill_ooo_finish_optimistic.result new file mode 100644 index 00000000000..2753e66fe48 --- /dev/null +++ b/mysql-test/suite/rpl/r/mdev-31448_kill_ooo_finish_optimistic.result @@ -0,0 +1,52 @@ +include/master-slave.inc +[connection master] +# MDEV-31448 OOO finish event group by killed worker +# Initialize test data +connection master; +call mtr.add_suppression("Slave: Connection was killed"); +call mtr.add_suppression("Slave: Commit failed due to failure of an earlier commit on which this one depends"); +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; +insert into t1 values (1); +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +include/stop_slave.inc +set @@global.slave_parallel_threads= 4; +set @@global.slave_parallel_mode= OPTIMISTIC; +set @@global.innodb_lock_wait_timeout= 30; +set @@global.slave_transaction_retries= 0; +connection slave1; +BEGIN; +SELECT * FROM t1 WHERE a=1 FOR UPDATE; +a +1 +connection master; +SET @old_dbug= @@SESSION.debug_dbug; +SET @@SESSION.debug_dbug="+d,binlog_force_commit_id"; +SET @commit_id= 10000; +update t1 set a=2 where a=1; +set statement skip_parallel_replication=1 for insert into t2 values (1); +drop table t2; +connection slave; +include/start_slave.inc +# wait for T1 +# wait for T2 +# wait for T3 +kill T2_TID; +connection slave1; +ROLLBACK; +connection master; +DROP TABLE t1; +include/save_master_gtid.inc +connection slave; +# +# Cleanup +include/stop_slave.inc +set @@global.slave_parallel_threads= 0; +set @@global.slave_parallel_mode= conservative; +set @@global.innodb_lock_wait_timeout= 50; +set @@global.slave_transaction_retries= 10; +include/start_slave.inc +include/sync_with_master_gtid.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_parallel_kill.result b/mysql-test/suite/rpl/r/rpl_parallel_kill.result new file mode 100644 index 00000000000..7e6b065725b --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_kill.result @@ -0,0 +1,142 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; +insert into t1 values (1); +include/save_master_gtid.inc +connection slave; +call mtr.add_suppression("Slave: Commit failed due to failure of an earlier commit on which this one depends"); +include/sync_with_master_gtid.inc +include/stop_slave.inc +set @save.slave_parallel_threads= @@global.slave_parallel_threads; +set @save.slave_parallel_mode= @@global.slave_parallel_mode; +set @@global.slave_parallel_threads= 3; +set @@global.slave_parallel_mode= CONSERVATIVE; +connection slave1; +BEGIN; +update t1 set a=2 where a=1; +connection master; +SET @old_dbug= @@SESSION.debug_dbug; +SET @@SESSION.debug_dbug="+d,binlog_force_commit_id"; +SET @commit_id= 10000; +update t1 set a=2 where a=1; +insert into t2 values (1); +SET @commit_id= 10001; +insert into t1 values (3); +connection slave; +include/start_slave.inc +kill $t3_tid; +connection slave1; +commit; +connection slave; +include/wait_for_slave_sql_to_stop.inc +update t1 set a=1 where a=2; +set @@global.slave_parallel_threads = @save.slave_parallel_threads; +set @@global.slave_parallel_mode = @save.slave_parallel_mode; +include/start_slave.inc +# +# Cleanup +connection master; +DROP TABLE t1, t2; +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +# MDEV-31448 OOO finish event group by killed worker +# Initialize test data +connection master; +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; +insert into t1 values (1); +include/save_master_gtid.inc +connection slave; +call mtr.add_suppression("Connection was killed"); +call mtr.add_suppression("Can.t find record"); +include/sync_with_master_gtid.inc +include/stop_slave.inc +set @save.slave_parallel_threads= @@global.slave_parallel_threads; +set @save.slave_parallel_mode= @@global.slave_parallel_mode; +set @@global.slave_parallel_threads= 3; +set @@global.slave_parallel_mode= OPTIMISTIC; +connection slave1; +begin; +update t1 set a=2 where a=1; +connection master; +set @old_dbug= @@session.debug_dbug; +set @@session.debug_dbug="+d,binlog_force_commit_id"; +set @commit_id= 10000; +update t1 set a=2 where a=1; +insert into t2 values (1); +drop table t2; +connection slave; +include/start_slave.inc +# wait for T1 +# wait for T2 +# wait for T3 +kill $t2_tid; +include/wait_for_slave_param.inc [Last_Errno] +connection slave1; +commit; +connection slave; +include/wait_for_slave_sql_to_stop.inc +update t1 set a=1 where a=2; +set @@global.slave_parallel_threads = @save.slave_parallel_threads; +set @@global.slave_parallel_mode = @save.slave_parallel_mode; +include/start_slave.inc +# +# Cleanup +connection master; +drop table t1; +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +# MDEV-31448 OOO finish event group by killed worker +# Initialize test data +connection master; +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; +insert into t1 values (1); +include/save_master_gtid.inc +connection slave; +call mtr.add_suppression("Connection was killed"); +call mtr.add_suppression("Can.t find record"); +include/sync_with_master_gtid.inc +include/stop_slave.inc +set @save.slave_parallel_threads= @@global.slave_parallel_threads; +set @save.slave_parallel_mode= @@global.slave_parallel_mode; +set @@global.slave_parallel_threads= 3; +set @@global.slave_parallel_mode= OPTIMISTIC; +connection slave1; +begin; +update t1 set a=2 where a=1; +connection master; +set @old_dbug= @@session.debug_dbug; +set @@session.debug_dbug="+d,binlog_force_commit_id"; +set @commit_id= 10000; +update t1 set a=2 where a=1; +set @commit_id= 10001; +set statement skip_parallel_replication=1 for insert into t2 values (1); +drop table t2; +connection slave; +include/start_slave.inc +# wait for T1 +# wait for T2 +# wait for T3 +kill $t2_tid; +include/wait_for_slave_param.inc [Last_Errno] +connection slave1; +commit; +connection slave; +include/wait_for_slave_sql_to_stop.inc +update t1 set a=1 where a=2; +set @@global.slave_parallel_threads = @save.slave_parallel_threads; +set @@global.slave_parallel_mode = @save.slave_parallel_mode; +include/start_slave.inc +# +# Cleanup +connection master; +drop table t1; +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/mdev-31448_kill_ooo_finish_optimistic.test b/mysql-test/suite/rpl/t/mdev-31448_kill_ooo_finish_optimistic.test new file mode 100644 index 00000000000..ae15ed64a65 --- /dev/null +++ b/mysql-test/suite/rpl/t/mdev-31448_kill_ooo_finish_optimistic.test @@ -0,0 +1,93 @@ +--source include/master-slave.inc +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_binlog_format_row.inc + +--echo # MDEV-31448 OOO finish event group by killed worker +# The test demonstrates how a killed worker access gco lists +# in finish_event_group() out-of-order to fire +# DBUG_ASSERT(!tmp_gco->next_gco || tmp_gco->last_sub_id > sub_id); +# in the buggy version. + +--echo # Initialize test data +--connection master +call mtr.add_suppression("Slave: Connection was killed"); +call mtr.add_suppression("Slave: Commit failed due to failure of an earlier commit on which this one depends"); +create table t1 (a int) engine=innodb; +create table t2 (a int) engine=innodb; + +insert into t1 values (1); +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +--let $save_slave_parallel_threads= `SELECT @@global.slave_parallel_threads` +--let $save_slave_parallel_mode= `SELECT @@global.slave_parallel_mode` +--let $save_innodb_lock_wait_timeout= `SELECT @@global.innodb_lock_wait_timeout` +--let $save_transaction_retries= `SELECT @@global.slave_transaction_retries` +set @@global.slave_parallel_threads= 4; +set @@global.slave_parallel_mode= OPTIMISTIC; +set @@global.innodb_lock_wait_timeout= 30; +set @@global.slave_transaction_retries= 0; + +--connection slave1 +BEGIN; +SELECT * FROM t1 WHERE a=1 FOR UPDATE; + +--connection master +SET @old_dbug= @@SESSION.debug_dbug; +SET @@SESSION.debug_dbug="+d,binlog_force_commit_id"; + +# GCO 1 +SET @commit_id= 10000; +# T1 +update t1 set a=2 where a=1; +# T2 +set statement skip_parallel_replication=1 for insert into t2 values (1); + +# GCO 2 +# T3 +drop table t2; + +--connection slave +--source include/start_slave.inc + +--echo # wait for T1 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Update_rows_log_event::find_row(-1)' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc + +--echo # wait for T2 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc +--let $t2_tid= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE 'Waiting for prior transaction to commit%' and command LIKE 'Slave_worker'` +--echo # wait for T3 +--let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%' and command LIKE 'Slave_worker'; +--source include/wait_condition.inc + +--replace_result $t2_tid T2_TID +--eval kill $t2_tid + +--sleep 1 + +--connection slave1 +# Release the blocked T1 +ROLLBACK; + +--connection master +DROP TABLE t1; +--source include/save_master_gtid.inc + +--connection slave +--echo # +--echo # Cleanup +--source include/stop_slave.inc +eval set @@global.slave_parallel_threads= $save_slave_parallel_threads; +eval set @@global.slave_parallel_mode= $save_slave_parallel_mode; +eval set @@global.innodb_lock_wait_timeout= $save_innodb_lock_wait_timeout; +eval set @@global.slave_transaction_retries= $save_transaction_retries; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +--source include/rpl_end.inc + diff --git a/mysql-test/suite/rpl/t/rpl_parallel_kill.test b/mysql-test/suite/rpl/t/rpl_parallel_kill.test new file mode 100644 index 00000000000..563b0aa61e9 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_kill.test @@ -0,0 +1,15 @@ +--source include/master-slave.inc +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_binlog_format_row.inc + +--source include/mdev-31448_conservative.inc + +--let $killed_trx_commits=1 +--source include/mdev-31448_optimistic.inc +--let $killed_trx_commits=0 +--source include/mdev-31448_optimistic.inc + + + +--source include/rpl_end.inc From 08585b0949c96935b5a7c2651d6d70ec5314af28 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 20 Jun 2023 23:23:26 +0200 Subject: [PATCH 14/43] MDEV-31509: Lost data with FTWRL and STOP SLAVE The largest_started_sub_id needs to be set under LOCK_parallel_entry together with testing stop_sub_id. However, in-between was the logic for do_ftwrl_wait(), which temporarily releases the mutex. This could lead to inconsistent stopping amongst worker threads and lost data. Fix by moving all the stop-related logic out from unrelated do_gco_wait() and do_ftwrl_wait() and into its own function do_stop_handling(). Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- .../suite/rpl/r/rpl_parallel_ftwrl.result | 105 +++++++++++++ .../suite/rpl/t/rpl_parallel_ftwrl.test | 143 ++++++++++++++++++ sql/rpl_parallel.cc | 77 ++++++++-- 3 files changed, 310 insertions(+), 15 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_ftwrl.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_ftwrl.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_ftwrl.result b/mysql-test/suite/rpl/r/rpl_parallel_ftwrl.result new file mode 100644 index 00000000000..b6c71055fe7 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_ftwrl.result @@ -0,0 +1,105 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=3; +SET @old_parallel_mode= @@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode=aggressive; +SET @old_dbug= @@GLOBAL.debug_dbug; +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +*** MDEV-31509: Lost data with FTWRL and STOP SLAVE +connection master; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (0,0); +INSERT INTO t2 VALUES (0,0); +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +connection slave; +*** Arrange for T1 to delay before entering GCO wait. +SET GLOBAL debug_dbug="+d,gco_wait_delay_gtid_0_x_99"; +*** Arrange for T2 to wait for FTWRL to start. +SET GLOBAL debug_dbug="+d,hold_worker_on_schedule"; +*** Arrange for T2 to delay wakeup from FTWRL pause. +SET GLOBAL debug_dbug="+d,delay_ftwrl_wait_gtid_0_x_100"; +connection master; +*** Event group T1 +SET SESSION gtid_seq_no=99; +INSERT INTO t1 VALUES (1,1); +connection slave; +*** 1a. Wait for T1 to be queued. +SET debug_sync="now WAIT_FOR gco_wait_paused"; +connection master; +*** Event group T2 +SET SESSION gtid_seq_no=100; +INSERT INTO t2 VALUES (2,2); +connection slave; +*** 1b. Wait for T2 to be queued. +SET debug_sync= "now WAIT_FOR reached_pause"; +connection slave1; +*** 2. Run FTWRL +SET GLOBAL debug_dbug= "+d,pause_for_ftwrl_wait"; +FLUSH TABLES WITH READ LOCK; +connection slave; +SET debug_sync= "now WAIT_FOR pause_ftwrl_waiting"; +*** 3. Wait for T2 to be waiting for FTWRL pause +SET debug_sync= "now SIGNAL continue_worker"; +*** 4. FTWRL completes, UNLOCK TABLES. +SET debug_sync="now SIGNAL pause_ftwrl_cont"; +connection slave1; +UNLOCK TABLES; +connection slave; +*** T2 is now ready to proceed after FTWRL pause, but did not wake up yet. +SET debug_sync="now WAIT_FOR pause_wait_started"; +*** 5. STOP SLAVE is run. +connection slave1; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; +STOP SLAVE; +connection slave; +SET debug_sync="now WAIT_FOR wait_for_done_waiting"; +*** 5. T2 wakes up after FTWRL pause, reaches wait_for_prior_commit(). +SET debug_sync="now SIGNAL pause_wait_continue"; +*** 6. T1 starts. +SET debug_sync="now SIGNAL gco_wait_cont"; +connection slave1; +connection slave; +include/wait_for_slave_to_stop.inc +connection master; +SELECT * FROM t1 ORDER BY a; +a b +0 0 +1 1 +SELECT * FROM t2 ORDER BY a; +a b +0 0 +2 2 +include/save_master_gtid.inc +connection slave; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT @@GLOBAL.gtid_slave_pos; +@@GLOBAL.gtid_slave_pos +0-1-100 +SELECT * FROM t1 ORDER BY a; +a b +0 0 +1 1 +SELECT * FROM t2 ORDER BY a; +a b +0 0 +2 2 +*** Clean up. +connection slave; +include/stop_slave.inc +SET DEBUG_SYNC= "RESET"; +SET GLOBAL slave_parallel_threads= @old_parallel_threads; +SET GLOBAL slave_parallel_mode= @old_parallel_mode; +SET GLOBAL debug_dbug=@old_dbug; +include/start_slave.inc +connection master; +DROP TABLE t1, t2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_ftwrl.test b/mysql-test/suite/rpl/t/rpl_parallel_ftwrl.test new file mode 100644 index 00000000000..308fcd0bd1f --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_ftwrl.test @@ -0,0 +1,143 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_binlog_format_mixed.inc +--source include/master-slave.inc + +--connection slave +--source include/stop_slave.inc +SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=3; +SET @old_parallel_mode= @@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode=aggressive; +SET @old_dbug= @@GLOBAL.debug_dbug; +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--echo *** MDEV-31509: Lost data with FTWRL and STOP SLAVE +# The bug was as follows: +# 1. Event groups T1 and T2 are queued but not started yet. +# 2. FLUSH TABLE WITH READ LOCKS starts, sets rpl_parallel_entry::pause_sub_id +# 3. T2 Sees pause_sub_id, goes to wait for the pause to complete. +# 4. FTWRL completes, UNLOCK TABLES is run. +# 5. STOP SLAVE is run, sets rpl_parallel_entry::stop_sub_id. +# 6. T2 wakes up after FTWRL pause, only now sets +# rpl_parallel_entry::largest_started_sub_id. This is the bug, +# largest_started_sub_id is set too late here. +# 7. T1 starts, it sees stop_sub_idLOCK_parallel_entry); } while (wait_count > entry->count_committing_event_groups); } +} - if (entry->force_abort && rgi->gtid_sub_id > entry->stop_sub_id) + +static bool +do_stop_handling(rpl_group_info *rgi) +{ + bool should_stop= false; + rpl_parallel_entry *entry= rgi->parallel_entry; + + mysql_mutex_assert_owner(&entry->LOCK_parallel_entry); + + if (unlikely(entry->force_abort) && rgi->gtid_sub_id > entry->stop_sub_id) { /* We are stopping (STOP SLAVE), and this event group need not be applied @@ -430,10 +440,26 @@ do_gco_wait(rpl_group_info *rgi, group_commit_orderer *gco, rather than execute, the following events. Once all queued events have been skipped, the STOP SLAVE is complete (for this thread). */ - return true; + should_stop= true; } - else - return false; + + if (unlikely(entry->stop_on_error_sub_id <= rgi->wait_commit_sub_id)) + { + rgi->worker_error= 1; + should_stop= true; + } + + if (likely(!should_stop)) + { + /* + Since we did not decide to stop, bump the largest_started_sub_id while + still holding LOCK_parallel_entry. + */ + if (rgi->gtid_sub_id > entry->largest_started_sub_id) + entry->largest_started_sub_id= rgi->gtid_sub_id; + } + + return should_stop; } @@ -480,15 +506,25 @@ do_ftwrl_wait(rpl_group_info *rgi, mysql_cond_wait(&entry->COND_parallel_entry, &entry->LOCK_parallel_entry); } while (sub_id > entry->pause_sub_id); + DBUG_EXECUTE_IF("delay_ftwrl_wait_gtid_0_x_100", { + if (rgi->current_gtid.domain_id == 0 && + rgi->current_gtid.seq_no == 100) { + /* + Simulate delayed wakeup from the mysql_cond_wait(). To do this, we + need to have the LOCK_parallel_entry mutex released during the wait. + */ + mysql_mutex_unlock(&entry->LOCK_parallel_entry); + debug_sync_set_action(thd, + STRING_WITH_LEN("now SIGNAL pause_wait_started WAIT_FOR pause_wait_continue")); + mysql_mutex_lock(&entry->LOCK_parallel_entry); + } + }); /* We do not call EXIT_COND() here, as this will be done later by our caller (since we set *did_enter_cond to true). */ } - if (sub_id > entry->largest_started_sub_id) - entry->largest_started_sub_id= sub_id; - DBUG_RETURN(aborted); } @@ -646,7 +682,17 @@ rpl_pause_for_ftwrl(THD *thd) mysql_mutex_unlock(&rpt->LOCK_rpl_thread); ++e->need_sub_id_signal; if (e->pause_sub_id == (uint64)ULONGLONG_MAX) + { e->pause_sub_id= e->largest_started_sub_id; + DBUG_EXECUTE_IF("pause_for_ftwrl_wait", { + mysql_mutex_unlock(&e->LOCK_parallel_entry); + debug_sync_set_action(thd, + STRING_WITH_LEN("now " + "SIGNAL pause_ftwrl_waiting " + "WAIT_FOR pause_ftwrl_cont")); + mysql_mutex_lock(&e->LOCK_parallel_entry); + }); + } thd->ENTER_COND(&e->COND_parallel_entry, &e->LOCK_parallel_entry, &stage_waiting_for_ftwrl_threads_to_pause, &old_stage); thd->set_time_for_next_stage(); @@ -1284,14 +1330,15 @@ handle_rpl_parallel_thread(void *arg) event_gtid_sub_id= rgi->gtid_sub_id; rgi->thd= thd; - mysql_mutex_lock(&entry->LOCK_parallel_entry); - skip_event_group= do_gco_wait(rgi, gco, &did_enter_cond, &old_stage); + DBUG_EXECUTE_IF("gco_wait_delay_gtid_0_x_99", { + if (rgi->current_gtid.domain_id == 0 && rgi->current_gtid.seq_no == 99) { + debug_sync_set_action(thd, + STRING_WITH_LEN("now SIGNAL gco_wait_paused WAIT_FOR gco_wait_cont")); + } }); - if (unlikely(entry->stop_on_error_sub_id <= rgi->wait_commit_sub_id)) - { - skip_event_group= true; - rgi->worker_error= 1; - } + mysql_mutex_lock(&entry->LOCK_parallel_entry); + do_gco_wait(rgi, gco, &did_enter_cond, &old_stage); + skip_event_group= do_stop_handling(rgi); if (likely(!skip_event_group)) skip_event_group= do_ftwrl_wait(rgi, &did_enter_cond, &old_stage); From 3f5cee8f541c3f9119bf37a3f4d7e58a46dd7e7c Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 21 Jun 2023 21:45:29 +0200 Subject: [PATCH 15/43] Fix one case that should not be marked transactional in the GTID event The case is statement format and mixed InnoDB/MyISAM without binlog_direct_non_trans_update. Fix due to Brandon Nesterenko. Reviewed-by: Andrei Elkin Signed-off-by: Kristian Nielsen --- sql/log_event.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index bafcf34cc2e..181a26ccdb5 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -8013,7 +8013,10 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg, thd_arg->transaction.all.has_created_dropped_temp_table() || thd_arg->transaction.all.trans_executed_admin_cmd()) flags2|= FL_DDL; - else if (is_transactional && !is_tmp_table) + else if (is_transactional && !is_tmp_table && + !(thd_arg->transaction.all.modified_non_trans_table && + thd->variables.binlog_direct_non_trans_update == 0 && + !thd->is_current_stmt_binlog_format_row())) flags2|= FL_TRANSACTIONAL; if (!(thd_arg->variables.option_bits & OPTION_RPL_SKIP_PARALLEL)) flags2|= FL_ALLOW_PARALLEL; From 7dde504aef06fee8aa335045cdb478062cd2c2a0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 5 Jul 2023 19:01:13 +0300 Subject: [PATCH 16/43] q# This is a combination of 2 commits. MDEV-31503 ALTER SEQUENCE ends up in optimistic parallel slave binlog out-of-order The OOO error still was possible even after MDEV-31077. This time it occured through open_table() when the sequence table was not in the table cache *and* the table was created before the last server restart. In such context a internal (read-only) transaction is committed and it was not blocked from doing a wakeup() call to subsequent transactions. Fixed with extending suspend_subsequent_commits() effect for the entirety of Sql_cmd_alter_sequence::execute(). An elaborated MDEV-31077 test proves the fixes of both failure scenarios. Also the bug condition suggests a workaround to pre-SELECT sequence tables before START SLAVE. Reviewed-by: Brandon Nesterenko --- .../suite/rpl/r/rpl_parallel_seq.result | 13 +++++- mysql-test/suite/rpl/t/rpl_parallel_seq.test | 21 +++++++++- sql/sql_sequence.cc | 41 ++++++++++++------- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_parallel_seq.result b/mysql-test/suite/rpl/r/rpl_parallel_seq.result index 60061049ed4..f5b062236db 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_seq.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_seq.result @@ -47,11 +47,21 @@ Warnings: Note 1255 Slave already has been stopped RESET MASTER; SET @@global.gtid_slave_pos=""; -SET @@global.gtid_strict_mode=1; connection master; RESET MASTER; CREATE TABLE ti (a INT) ENGINE=innodb; CREATE SEQUENCE s2 ENGINE=innodb; +include/save_master_gtid.inc +connection slave; +include/start_slave.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +include/rpl_restart_server.inc [server_number=2] +SET @@global.slave_parallel_threads=2; +SET @@global.slave_parallel_mode=optimistic; +SET @@global.debug_dbug="+d,hold_worker_on_schedule"; +SET @@global.gtid_strict_mode=1; +connection master; SET @@gtid_seq_no=100; ALTER SEQUENCE s2 restart with 1; INSERT INTO ti SET a=1; @@ -60,6 +70,7 @@ SELECT @@global.gtid_binlog_state "Master gtid state"; Master gtid state 0-1-101 connection slave; +SET STATEMENT sql_log_bin=0 FOR FLUSH TABLES; include/start_slave.inc SELECT @@global.gtid_binlog_state, @@global.gtid_slave_pos as "no 100,101 yet in both"; @@global.gtid_binlog_state no 100,101 yet in both diff --git a/mysql-test/suite/rpl/t/rpl_parallel_seq.test b/mysql-test/suite/rpl/t/rpl_parallel_seq.test index 741859bc588..e975d079e52 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_seq.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_seq.test @@ -77,15 +77,28 @@ SET DEBUG_SYNC = 'now SIGNAL continue_worker'; --source include/stop_slave.inc RESET MASTER; SET @@global.gtid_slave_pos=""; ---let $slave_gtid_strict_mode=`select @@global.gtid_strict_mode` -SET @@global.gtid_strict_mode=1; --connection master RESET MASTER; # Load from master CREATE TABLE ti (a INT) ENGINE=innodb; CREATE SEQUENCE s2 ENGINE=innodb; +--source include/save_master_gtid.inc +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +--let $rpl_server_number= 2 +--source include/rpl_restart_server.inc +# upon restart +SET @@global.slave_parallel_threads=2; +SET @@global.slave_parallel_mode=optimistic; +SET @@global.debug_dbug="+d,hold_worker_on_schedule"; +--let $slave_gtid_strict_mode=`select @@global.gtid_strict_mode` +SET @@global.gtid_strict_mode=1; + +--connection master SET @@gtid_seq_no=100; ALTER SEQUENCE s2 restart with 1; INSERT INTO ti SET a=1; @@ -93,6 +106,10 @@ INSERT INTO ti SET a=1; SELECT @@global.gtid_binlog_state "Master gtid state"; --connection slave +# The following FT complicates the opening table time with committing +# an internal transaction. The rest of the test also proves +# MDEV-31503 "branch" of the OOO error is fixed. +SET STATEMENT sql_log_bin=0 FOR FLUSH TABLES; --source include/start_slave.inc --let $wait_condition= SELECT count(*) = 1 FROM information_schema.processlist WHERE state LIKE "Waiting for prior transaction to commit" diff --git a/sql/sql_sequence.cc b/sql/sql_sequence.cc index 2904749d1e6..4ba1e6092bd 100644 --- a/sql/sql_sequence.cc +++ b/sql/sql_sequence.cc @@ -900,6 +900,20 @@ end: DBUG_RETURN(error); } +#if defined(HAVE_REPLICATION) +class wait_for_commit_raii +{ +private: + THD *m_thd; + wait_for_commit *m_wfc; + +public: + wait_for_commit_raii(THD* thd) : + m_thd(thd), m_wfc(thd->suspend_subsequent_commits()) + {} + ~wait_for_commit_raii() { m_thd->resume_subsequent_commits(m_wfc); } +}; +#endif bool Sql_cmd_alter_sequence::execute(THD *thd) { @@ -912,7 +926,10 @@ bool Sql_cmd_alter_sequence::execute(THD *thd) SEQUENCE *seq; No_such_table_error_handler no_such_table_handler; DBUG_ENTER("Sql_cmd_alter_sequence::execute"); - +#if defined(HAVE_REPLICATION) + /* No wakeup():s of subsequent commits is allowed in this function. */ + wait_for_commit_raii suspend_wfc(thd); +#endif if (check_access(thd, ALTER_ACL, first_table->db.str, &first_table->grant.privilege, @@ -1010,19 +1027,15 @@ bool Sql_cmd_alter_sequence::execute(THD *thd) else table->file->print_error(error, MYF(0)); table->s->sequence->write_unlock(table); - { - wait_for_commit* suspended_wfc= thd->suspend_subsequent_commits(); - if (trans_commit_stmt(thd)) - error= 1; - if (trans_commit_implicit(thd)) - error= 1; - thd->resume_subsequent_commits(suspended_wfc); - DBUG_EXECUTE_IF("hold_worker_on_schedule", - { - /* delay binlogging of a parent trx in rpl_parallel_seq */ - my_sleep(100000); - }); - } + if (trans_commit_stmt(thd)) + error= 1; + if (trans_commit_implicit(thd)) + error= 1; + DBUG_EXECUTE_IF("hold_worker_on_schedule", + { + /* delay binlogging of a parent trx in rpl_parallel_seq */ + my_sleep(100000); + }); if (likely(!error)) error= write_bin_log(thd, 1, thd->query(), thd->query_length()); if (likely(!error)) From e1d31a10afcca0d8e16f35c8bb4c25c23f94f136 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 13 Jul 2023 13:22:24 +1000 Subject: [PATCH 17/43] MDEV-31524 Fixing spider table param / variable overriding The existing (incorrect) overriding mechanism is: Non-minus-one var value overrides table param overrides default value. Before MDEV-27169, unspecified var value is -1. So if the user sets both the var to be a value other than -1 and the table param, the var value will prevail, which is incorrect. After MDEV-27169, unspecified var value is default value. So if the user does not set the var but sets the table param, the default value will prevail, which is even more incorrect. This patch fixes it so that table param, if specified, always overrides var value, and the latter if not specified or set to -1, falls back to the default value We achieve this by replacing all such overriding in spd_param.cc with macros that override in the correct way, and removing all the "overriding -1" lines involving table params in spider_set_connect_info_default() except for those table params not defined as sysvar/thdvar in spd_params.cc We also introduced macros for non-overriding sysvar and thdvar, so that the code is cleaner and less error-prone In server versions where MDEV-27169 has not been applied, we also backport the patch, that is, replacing -1 default values with real default values In server versions where MDEV-28006 has not been applied, we do the same for udf params --- .../spider/bugfix/r/mdev_31524.result | 46 + .../spider/bugfix/t/mdev_31524.test | 73 + storage/spider/spd_copy_tables.cc | 6 - storage/spider/spd_db_conn.cc | 4 +- storage/spider/spd_direct_sql.cc | 15 - storage/spider/spd_param.cc | 1266 +++++------------ storage/spider/spd_table.cc | 146 -- 7 files changed, 449 insertions(+), 1107 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result new file mode 100644 index 00000000000..645fc62862d --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result @@ -0,0 +1,46 @@ + +MDEV-31524 Spider variables that double as table params overriding mechanism is buggy + +for master_1 +for child2 +for child3 +SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +set session spider_read_only_mode = default; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 1 */ insert into t1 values (42); +drop table t1, t2; +set session spider_read_only_mode = 1; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 2 */ insert into t1 values (42); +ERROR HY000: Table 'test.t1' is read only +drop table t1, t2; +set session spider_read_only_mode = -1; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 3 */ insert into t1 values (42); +drop table t1, t2; +SET session spider_read_only_mode = default; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 4 */ insert into t1 values (42); +ERROR HY000: Table 'test.t1' is read only +drop table t1, t2; +set session spider_read_only_mode = 1; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 5 */ insert into t1 values (42); +drop table t1, t2; +SET session spider_read_only_mode = -1; +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "srv_mdev_31524",TABLE "t2"'; +/* 6 */ insert into t1 values (42); +ERROR HY000: Table 'test.t1' is read only +drop table t1, t2; +drop server srv_mdev_31524; +SET session spider_read_only_mode = @old_spider_read_only_mode; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test new file mode 100644 index 00000000000..64cbf4154aa --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test @@ -0,0 +1,73 @@ +--echo +--echo MDEV-31524 Spider variables that double as table params overriding mechanism is buggy +--echo + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--let $srv=srv_mdev_31524 +SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +# when the user does not set var nor the table option, the default +# value (0 in this case) takes effect. +set session spider_read_only_mode = default; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"'; +/* 1 */ insert into t1 values (42); +drop table t1, t2; + +# when the user sets var but not the table option, the var should be +# take effect. +set session spider_read_only_mode = 1; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"'; +--error 12518 +/* 2 */ insert into t1 values (42); +drop table t1, t2; + +# when the user sets var to -1 and does not set the table option, the +# default value takes effect. +set session spider_read_only_mode = -1; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t2"'; +/* 3 */ insert into t1 values (42); +drop table t1, t2; + +# when the user does not set the var, but sets the table option, the +# table option should take effect +SET session spider_read_only_mode = default; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "$srv",TABLE "t2"'; +--error 12518 +/* 4 */ insert into t1 values (42); +drop table t1, t2; + +# when the user sets both var and table option, the table option +# should take precedence +set session spider_read_only_mode = 1; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "0", WRAPPER "mysql", srv "$srv",TABLE "t2"'; +/* 5 */ insert into t1 values (42); +drop table t1, t2; + +# when the user sets the var to -1 and sets the table option, the +# table option should take effect +SET session spider_read_only_mode = -1; +create table t2 (c int); +eval create table t1 (c int) ENGINE=Spider COMMENT='read_only_mode "1", WRAPPER "mysql", srv "$srv",TABLE "t2"'; +--error 12518 +/* 6 */ insert into t1 values (42); +drop table t1, t2; + +eval drop server $srv; + +SET session spider_read_only_mode = @old_spider_read_only_mode; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/spd_copy_tables.cc b/storage/spider/spd_copy_tables.cc index 319b02462b1..e00e0608776 100644 --- a/storage/spider/spd_copy_tables.cc +++ b/storage/spider/spd_copy_tables.cc @@ -64,12 +64,6 @@ int spider_udf_set_copy_tables_param_default( } } - if (copy_tables->bulk_insert_interval == -1) - copy_tables->bulk_insert_interval = 10; - if (copy_tables->bulk_insert_rows == -1) - copy_tables->bulk_insert_rows = 100; - if (copy_tables->use_table_charset == -1) - copy_tables->use_table_charset = 1; if (copy_tables->use_transaction == -1) copy_tables->use_transaction = 1; #ifndef WITHOUT_SPIDER_BG_SEARCH diff --git a/storage/spider/spd_db_conn.cc b/storage/spider/spd_db_conn.cc index 8712dc91a65..de85e0b99b2 100644 --- a/storage/spider/spd_db_conn.cc +++ b/storage/spider/spd_db_conn.cc @@ -11559,7 +11559,7 @@ int spider_db_udf_copy_tables( error_num = result->get_errno(); if (error_num == HA_ERR_END_OF_FILE) { - if (roop_count < copy_tables->bulk_insert_rows) + if (roop_count < bulk_insert_rows) { end_of_file = TRUE; if (roop_count) @@ -11583,8 +11583,6 @@ int spider_db_udf_copy_tables( pthread_mutex_unlock(&tmp_conn->mta_conn_mutex); goto error_db_query; } - bulk_insert_rows = spider_param_udf_ct_bulk_insert_rows( - copy_tables->bulk_insert_rows); if ( select_ct->append_key_order_str(key_info, 0, FALSE) || select_ct->append_limit(0, bulk_insert_rows) || diff --git a/storage/spider/spd_direct_sql.cc b/storage/spider/spd_direct_sql.cc index 0574c18b078..6a04488c50f 100644 --- a/storage/spider/spd_direct_sql.cc +++ b/storage/spider/spd_direct_sql.cc @@ -1451,25 +1451,10 @@ int spider_udf_set_direct_sql_param_default( } } - if (direct_sql->table_loop_mode == -1) - direct_sql->table_loop_mode = 0; if (direct_sql->priority == -1) direct_sql->priority = 1000000; - if (direct_sql->connect_timeout == -1) - direct_sql->connect_timeout = 6; - if (direct_sql->net_read_timeout == -1) - direct_sql->net_read_timeout = 600; - if (direct_sql->net_write_timeout == -1) - direct_sql->net_write_timeout = 600; - if (direct_sql->bulk_insert_rows == -1) - direct_sql->bulk_insert_rows = 3000; if (direct_sql->connection_channel == -1) direct_sql->connection_channel = 0; -#if MYSQL_VERSION_ID < 50500 -#else - if (direct_sql->use_real_table == -1) - direct_sql->use_real_table = 0; -#endif if (direct_sql->error_rw_mode == -1) direct_sql->error_rw_mode = 0; for (roop_count = 0; roop_count < direct_sql->table_count; roop_count++) diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 81540a1ef91..b77acef57fe 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -1,4 +1,5 @@ /* Copyright (C) 2008-2018 Kentoku Shiba + Copyright (C) 2023 MariaDB plc This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -13,6 +14,26 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ +/** + @file + + There are several kinds of spider parameters. + + - sysvar/thdvar that are not table parameters. These variables do + not appear in a `SPIDER_SHARE`. Examples include `support_xa` and + `conn_recycle_mode`. Their values are commonly retrieved by + `SPIDER_SYSVAR_VALUE_FUNC()` and `SPIDER_THDVAR_VALUE_FUNC()` + - sysvar/thdvar that are also table parameters. These variables + commonly appear in a `SPIDER_SHARE`. Examples include + `read_only_mode` and `use_table_charset`. Table parameter values + override variable values, and their values are commonly retrieved + by `SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC()` and + `SPIDER_THDVAR_OVERRIDE_VALUE_FUNC()`. + - table parameters that are not sysvar/thdvar. Examples include + host and username. They are not handled in this file which is only + concerned with global/session variables +*/ + #define MYSQL_SERVER 1 #include #include "mysql_version.h" @@ -42,6 +63,57 @@ extern struct st_maria_plugin spider_i_s_alloc_mem_maria; extern volatile ulonglong spider_mon_table_cache_version; extern volatile ulonglong spider_mon_table_cache_version_req; +/* + Define a function returning the value of a global variable. +*/ +#define SPIDER_SYSVAR_VALUE_FUNC(param_type, param_name) \ + param_type spider_param_ ## param_name() \ + { \ + return SYSVAR(param_name); \ + } + +/* + Define a function returning the value of a session variable. +*/ +#define SPIDER_THDVAR_VALUE_FUNC(param_type, param_name) \ + param_type spider_param_ ## param_name(THD *thd) \ + { \ + return THDVAR(thd, param_name); \ + } + +/* + Define a function returning the value of a table param that is also a + global variable. + + If the table param value is not -1, use the table param value. + Otherwise if the variable value is not -1, use the variable value. + Otherwise use the default variable value. +*/ +#define SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(param_type, param_name) \ + param_type spider_param_ ## param_name(param_type param_name) \ + { \ + return param_name != -1 ? param_name : \ + SYSVAR(param_name) != -1 ? SYSVAR(param_name) : \ + MYSQL_SYSVAR_NAME(param_name).def_val; \ + } + +/* + Define a function returning the value of a table param that is also a + session variable. + + If the table param value is not -1, use the table param value. + Otherwise if the variable value is not -1, use the variable value. + Otherwise use the default variable value. +*/ +#define SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(param_type, param_name) \ + param_type spider_param_ ## param_name(THD* thd, \ + param_type param_name) \ + { \ + return param_name != -1 ? param_name : \ + THDVAR(thd, param_name) != -1 ? THDVAR(thd, param_name) : \ + MYSQL_SYSVAR_NAME(param_name).def_val; \ + } + #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) { @@ -172,11 +244,7 @@ static MYSQL_SYSVAR_BOOL( TRUE ); -my_bool spider_param_support_xa() -{ - DBUG_ENTER("spider_param_support_xa"); - DBUG_RETURN(spider_support_xa); -} +SPIDER_SYSVAR_VALUE_FUNC(my_bool, support_xa) static my_bool spider_connect_mutex; static MYSQL_SYSVAR_BOOL( @@ -189,11 +257,7 @@ static MYSQL_SYSVAR_BOOL( FALSE ); -my_bool spider_param_connect_mutex() -{ - DBUG_ENTER("spider_param_connect_mutex"); - DBUG_RETURN(spider_connect_mutex); -} +SPIDER_SYSVAR_VALUE_FUNC(my_bool, connect_mutex) static uint spider_connect_error_interval; /* @@ -212,11 +276,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_connect_error_interval() -{ - DBUG_ENTER("spider_param_connect_error_interval"); - DBUG_RETURN(spider_connect_error_interval); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, connect_error_interval) static uint spider_table_init_error_interval; /* @@ -235,15 +295,11 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_table_init_error_interval() -{ - DBUG_ENTER("spider_param_table_init_error_interval"); - DBUG_RETURN(spider_table_init_error_interval); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, table_init_error_interval) static int spider_use_table_charset; /* - -1 :use table parameter + -1 :fallback to default 0 :use utf8 1 :use table charset */ @@ -254,19 +310,13 @@ static MYSQL_SYSVAR_INT( "Use table charset for remote access", NULL, NULL, - -1, + 1, -1, 1, 0 ); -int spider_param_use_table_charset( - int use_table_charset -) { - DBUG_ENTER("spider_param_use_table_charset"); - DBUG_RETURN(spider_use_table_charset == -1 ? - use_table_charset : spider_use_table_charset); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, use_table_charset) /* 0: no recycle @@ -285,12 +335,7 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_conn_recycle_mode( - THD *thd -) { - DBUG_ENTER("spider_param_conn_recycle_mode"); - DBUG_RETURN(THDVAR(thd, conn_recycle_mode)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, conn_recycle_mode) /* 0: weak @@ -308,12 +353,7 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_conn_recycle_strict( - THD *thd -) { - DBUG_ENTER("spider_param_conn_recycle_strict"); - DBUG_RETURN(THDVAR(thd, conn_recycle_strict)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, conn_recycle_strict) /* FALSE: no sync @@ -328,12 +368,7 @@ static MYSQL_THDVAR_BOOL( TRUE /* def */ ); -bool spider_param_sync_trx_isolation( - THD *thd -) { - DBUG_ENTER("spider_param_sync_trx_isolation"); - DBUG_RETURN(THDVAR(thd, sync_trx_isolation)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, sync_trx_isolation) /* FALSE: no use @@ -348,12 +383,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_use_consistent_snapshot( - THD *thd -) { - DBUG_ENTER("spider_param_use_consistent_snapshot"); - DBUG_RETURN(THDVAR(thd, use_consistent_snapshot)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, use_consistent_snapshot) /* FALSE: off @@ -368,12 +398,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_internal_xa( - THD *thd -) { - DBUG_ENTER("spider_param_internal_xa"); - DBUG_RETURN(THDVAR(thd, internal_xa)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, internal_xa) /* 0 :err when use a spider table @@ -393,12 +418,7 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_internal_xa_snapshot( - THD *thd -) { - DBUG_ENTER("spider_param_internal_xa_snapshot"); - DBUG_RETURN(THDVAR(thd, internal_xa_snapshot)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, internal_xa_snapshot) /* 0 :off @@ -417,12 +437,7 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_force_commit( - THD *thd -) { - DBUG_ENTER("spider_param_force_commit"); - DBUG_RETURN(THDVAR(thd, force_commit)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, force_commit) /* 0: register all XA transaction @@ -440,15 +455,10 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_xa_register_mode( - THD *thd -) { - DBUG_ENTER("spider_param_xa_register_mode"); - DBUG_RETURN(THDVAR(thd, xa_register_mode)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, xa_register_mode) /* - -1 :use table parameter + -1 :fallback to default 0-:offset */ static MYSQL_THDVAR_LONGLONG( @@ -457,23 +467,16 @@ static MYSQL_THDVAR_LONGLONG( "Internal offset", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_internal_offset( - THD *thd, - longlong internal_offset -) { - DBUG_ENTER("spider_param_internal_offset"); - DBUG_RETURN(THDVAR(thd, internal_offset) < 0 ? - internal_offset : THDVAR(thd, internal_offset)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, internal_offset) /* - -1 :use table parameter + -1 :fallback to default 0-:limit */ static MYSQL_THDVAR_LONGLONG( @@ -482,23 +485,16 @@ static MYSQL_THDVAR_LONGLONG( "Internal limit", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 9223372036854775807LL, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_internal_limit( - THD *thd, - longlong internal_limit -) { - DBUG_ENTER("spider_param_internal_limit"); - DBUG_RETURN(THDVAR(thd, internal_limit) < 0 ? - internal_limit : THDVAR(thd, internal_limit)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, internal_limit) /* - -1 :use table parameter + -1 :fallback to default 0-:number of rows at a select */ static MYSQL_THDVAR_LONGLONG( @@ -507,23 +503,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of rows at a select", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 9223372036854775807LL, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_split_read( - THD *thd, - longlong split_read -) { - DBUG_ENTER("spider_param_split_read"); - DBUG_RETURN(THDVAR(thd, split_read) < 0 ? - split_read : THDVAR(thd, split_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, split_read) /* - -1 :use table parameter + -1 :fallback to default 0 :doesn't use "offset" and "limit" for "split_read" 1-:magnification */ @@ -533,23 +522,16 @@ static MYSQL_THDVAR_INT( "Use offset and limit parameter in SQL for split_read parameter.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -double spider_param_semi_split_read( - THD *thd, - double semi_split_read -) { - DBUG_ENTER("spider_param_semi_split_read"); - DBUG_RETURN(THDVAR(thd, semi_split_read) < 0 ? - semi_split_read : THDVAR(thd, semi_split_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(double, semi_split_read) /* - -1 :use table parameter + -1 :fallback to default 0-:the limit value */ static MYSQL_THDVAR_LONGLONG( @@ -558,23 +540,16 @@ static MYSQL_THDVAR_LONGLONG( "The limit value for semi_split_read", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 9223372036854775807LL, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_semi_split_read_limit( - THD *thd, - longlong semi_split_read_limit -) { - DBUG_ENTER("spider_param_semi_split_read_limit"); - DBUG_RETURN(THDVAR(thd, semi_split_read_limit) < 0 ? - semi_split_read_limit : THDVAR(thd, semi_split_read_limit)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, semi_split_read_limit) /* - -1 :use table parameter + -1 :fallback to default 0 :no alloc 1-:alloc size */ @@ -584,23 +559,16 @@ static MYSQL_THDVAR_INT( "Initial sql string alloc size", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1024, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_init_sql_alloc_size( - THD *thd, - int init_sql_alloc_size -) { - DBUG_ENTER("spider_param_init_sql_alloc_size"); - DBUG_RETURN(THDVAR(thd, init_sql_alloc_size) < 0 ? - init_sql_alloc_size : THDVAR(thd, init_sql_alloc_size)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, init_sql_alloc_size) /* - -1 :use table parameter + -1 :fallback to default 0 :off 1 :on */ @@ -610,24 +578,17 @@ static MYSQL_THDVAR_INT( "Reset sql string alloc after execute", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_reset_sql_alloc( - THD *thd, - int reset_sql_alloc -) { - DBUG_ENTER("spider_param_reset_sql_alloc"); - DBUG_RETURN(THDVAR(thd, reset_sql_alloc) < 0 ? - reset_sql_alloc : THDVAR(thd, reset_sql_alloc)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, reset_sql_alloc) #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) /* - -1 :use table parameter + -1 :fallback to default 0-:result free size for handlersocket */ static MYSQL_THDVAR_LONGLONG( @@ -636,24 +597,17 @@ static MYSQL_THDVAR_LONGLONG( "Result free size for handlersocket", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1048576, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_hs_result_free_size( - THD *thd, - longlong hs_result_free_size -) { - DBUG_ENTER("spider_param_hs_result_free_size"); - DBUG_RETURN(THDVAR(thd, hs_result_free_size) < 0 ? - hs_result_free_size : THDVAR(thd, hs_result_free_size)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUN(longlong, hs_result_free_size) #endif /* - -1 :use table parameter + -1 :fallback to default 0 :off 1 :on */ @@ -663,23 +617,16 @@ static MYSQL_THDVAR_INT( "Sprit read mode for multi range", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 100, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_multi_split_read( - THD *thd, - int multi_split_read -) { - DBUG_ENTER("spider_param_multi_split_read"); - DBUG_RETURN(THDVAR(thd, multi_split_read) < 0 ? - multi_split_read : THDVAR(thd, multi_split_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, multi_split_read) /* - -1 :use table parameter + -1 :fallback to default 0-:max order columns */ static MYSQL_THDVAR_INT( @@ -688,20 +635,13 @@ static MYSQL_THDVAR_INT( "Max columns for order by", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 32767, /* def */ -1, /* min */ 32767, /* max */ 0 /* blk */ ); -int spider_param_max_order( - THD *thd, - int max_order -) { - DBUG_ENTER("spider_param_max_order"); - DBUG_RETURN(THDVAR(thd, max_order) < 0 ? - max_order : THDVAR(thd, max_order)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, max_order) /* -1 :off @@ -722,12 +662,7 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -int spider_param_semi_trx_isolation( - THD *thd -) { - DBUG_ENTER("spider_param_semi_trx_isolation"); - DBUG_RETURN(THDVAR(thd, semi_trx_isolation)); -} +SPIDER_THDVAR_VALUE_FUNC(int, semi_trx_isolation) static int spider_param_semi_table_lock_check( MYSQL_THD thd, @@ -779,19 +714,13 @@ static MYSQL_THDVAR_INT( "Table lock during execute a sql", /* comment */ &spider_param_semi_table_lock_check, /* check */ NULL, /* update */ - 1, /* def */ + 0, /* def */ 0, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_semi_table_lock( - THD *thd, - int semi_table_lock -) { - DBUG_ENTER("spider_param_semi_table_lock"); - DBUG_RETURN((semi_table_lock & THDVAR(thd, semi_table_lock))); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, semi_table_lock) static int spider_param_semi_table_lock_connection_check( MYSQL_THD thd, @@ -844,20 +773,13 @@ static MYSQL_THDVAR_INT( "Use different connection if semi_table_lock is enabled", /* comment */ &spider_param_semi_table_lock_connection_check, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_semi_table_lock_connection( - THD *thd, - int semi_table_lock_connection -) { - DBUG_ENTER("spider_param_semi_table_lock_connection"); - DBUG_RETURN(THDVAR(thd, semi_table_lock_connection) == -1 ? - semi_table_lock_connection : THDVAR(thd, semi_table_lock_connection)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, semi_table_lock_connection) /* 0-:block_size @@ -874,15 +796,10 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_block_size( - THD *thd -) { - DBUG_ENTER("spider_param_block_size"); - DBUG_RETURN(THDVAR(thd, block_size)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, block_size) /* - -1 :use table parameter + -1 :fallback to default 0 :off 1 :lock in share mode 2 :for update @@ -893,20 +810,13 @@ static MYSQL_THDVAR_INT( "Lock for select with update", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_selupd_lock_mode( - THD *thd, - int selupd_lock_mode -) { - DBUG_ENTER("spider_param_selupd_lock_mode"); - DBUG_RETURN(THDVAR(thd, selupd_lock_mode) == -1 ? - selupd_lock_mode : THDVAR(thd, selupd_lock_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, selupd_lock_mode) /* FALSE: no sync @@ -921,12 +831,7 @@ static MYSQL_THDVAR_BOOL( TRUE /* def */ ); -bool spider_param_sync_autocommit( - THD *thd -) { - DBUG_ENTER("spider_param_sync_autocommit"); - DBUG_RETURN(THDVAR(thd, sync_autocommit)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, sync_autocommit) /* FALSE: not use @@ -941,12 +846,7 @@ static MYSQL_THDVAR_BOOL( TRUE /* def */ ); -bool spider_param_use_default_database( - THD *thd -) { - DBUG_ENTER("spider_param_use_default_database"); - DBUG_RETURN(THDVAR(thd, use_default_database)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, use_default_database) /* -1 :don't know or does not matter; don't send 'SET SQL_LOG_OFF' statement @@ -965,15 +865,10 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -int spider_param_internal_sql_log_off( - THD *thd -) { - DBUG_ENTER("spider_param_internal_sql_log_off"); - DBUG_RETURN(THDVAR(thd, internal_sql_log_off)); -} +SPIDER_THDVAR_VALUE_FUNC(int, internal_sql_log_off) /* - -1 :use table parameter + -1 :fallback to default 0-:bulk insert size */ static MYSQL_THDVAR_INT( @@ -982,23 +877,16 @@ static MYSQL_THDVAR_INT( "Bulk insert size", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 16000, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_bulk_size( - THD *thd, - int bulk_size -) { - DBUG_ENTER("spider_param_bulk_size"); - DBUG_RETURN(THDVAR(thd, bulk_size) < 0 ? - bulk_size : THDVAR(thd, bulk_size)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bulk_size) /* - -1 :use table parameter + -1 :fallback to default 0 : Send "update" and "delete" statements one by one. 1 : Send collected multiple "update" and "delete" statements. (Collected statements are sent one by one) @@ -1011,23 +899,16 @@ static MYSQL_THDVAR_INT( "The mode of bulk updating and deleting", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_bulk_update_mode( - THD *thd, - int bulk_update_mode -) { - DBUG_ENTER("spider_param_bulk_update_mode"); - DBUG_RETURN(THDVAR(thd, bulk_update_mode) == -1 ? - bulk_update_mode : THDVAR(thd, bulk_update_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bulk_update_mode) /* - -1 :use table parameter + -1 :fallback to default 0-:bulk update size */ static MYSQL_THDVAR_INT( @@ -1036,23 +917,24 @@ static MYSQL_THDVAR_INT( "Bulk update size", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 16000, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_bulk_update_size( - THD *thd, - int bulk_update_size -) { - DBUG_ENTER("spider_param_bulk_update_size"); - DBUG_RETURN(THDVAR(thd, bulk_update_size) == -1 ? - bulk_update_size : THDVAR(thd, bulk_update_size)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bulk_update_size) /* - -1 :use table parameter + Notes on merge conflicts (remove after merging): + 10.5: 48faa20db848012e2187a09e05aba832078cb82e + 10.6: 51ff9eddf7c0aaf1e022fcb3b48ec36835df7785 + 10.9: 06a61b8e453126c2de1649073f247d34e85f9702 + 10.10: 90cd0c156f5bb53fd058d2bbfb83f850ffae6722 + 10.11+: 124eb662700708f3c4b0fb77968f8b854d6bb4aa +*/ +/* + -1 :fallback to default 0 :off 1 :on */ @@ -1062,23 +944,16 @@ static MYSQL_THDVAR_INT( "Execute optimize to remote server", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_internal_optimize( - THD *thd, - int internal_optimize -) { - DBUG_ENTER("spider_param_internal_optimize"); - DBUG_RETURN(THDVAR(thd, internal_optimize) == -1 ? - internal_optimize : THDVAR(thd, internal_optimize)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, internal_optimize) /* - -1 :use table parameter + -1 :fallback to default 0 :off 1 :on */ @@ -1088,20 +963,13 @@ static MYSQL_THDVAR_INT( "Execute optimize to remote server with local", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_internal_optimize_local( - THD *thd, - int internal_optimize_local -) { - DBUG_ENTER("spider_param_internal_optimize_local"); - DBUG_RETURN(THDVAR(thd, internal_optimize_local) == -1 ? - internal_optimize_local : THDVAR(thd, internal_optimize_local)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, internal_optimize_local) /* FALSE: off @@ -1116,12 +984,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_use_flash_logs( - THD *thd -) { - DBUG_ENTER("spider_param_use_flash_logs"); - DBUG_RETURN(THDVAR(thd, use_flash_logs)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, use_flash_logs) /* 0 :off @@ -1140,12 +1003,7 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -int spider_param_use_snapshot_with_flush_tables( - THD *thd -) { - DBUG_ENTER("spider_param_use_snapshot_with_flush_tables"); - DBUG_RETURN(THDVAR(thd, use_snapshot_with_flush_tables)); -} +SPIDER_THDVAR_VALUE_FUNC(int, use_snapshot_with_flush_tables) /* FALSE: off @@ -1160,12 +1018,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_use_all_conns_snapshot( - THD *thd -) { - DBUG_ENTER("spider_param_use_all_conns_snapshot"); - DBUG_RETURN(THDVAR(thd, use_all_conns_snapshot)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, use_all_conns_snapshot) /* FALSE: off @@ -1180,12 +1033,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_lock_exchange( - THD *thd -) { - DBUG_ENTER("spider_param_lock_exchange"); - DBUG_RETURN(THDVAR(thd, lock_exchange)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, lock_exchange) /* FALSE: off @@ -1200,12 +1048,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_internal_unlock( - THD *thd -) { - DBUG_ENTER("spider_param_internal_unlock"); - DBUG_RETURN(THDVAR(thd, internal_unlock)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, internal_unlock) /* FALSE: off @@ -1220,15 +1063,10 @@ static MYSQL_THDVAR_BOOL( TRUE /* def */ ); -bool spider_param_semi_trx( - THD *thd -) { - DBUG_ENTER("spider_param_semi_trx"); - DBUG_RETURN(THDVAR(thd, semi_trx)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, semi_trx) /* - -1 :use table parameter + -1 :fallback to default 0-:seconds of timeout */ static MYSQL_THDVAR_INT( @@ -1237,25 +1075,16 @@ static MYSQL_THDVAR_INT( "Wait timeout of connecting to remote server", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 6, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_connect_timeout( - THD *thd, - int connect_timeout -) { - DBUG_ENTER("spider_param_connect_timeout"); - if (thd) - DBUG_RETURN(THDVAR(thd, connect_timeout) == -1 ? - connect_timeout : THDVAR(thd, connect_timeout)); - DBUG_RETURN(connect_timeout); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, connect_timeout) /* - -1 :use table parameter + -1 :fallback to default 0-:seconds of timeout */ static MYSQL_THDVAR_INT( @@ -1264,25 +1093,16 @@ static MYSQL_THDVAR_INT( "Wait timeout of receiving data from remote server", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 600, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_net_read_timeout( - THD *thd, - int net_read_timeout -) { - DBUG_ENTER("spider_param_net_read_timeout"); - if (thd) - DBUG_RETURN(THDVAR(thd, net_read_timeout) == -1 ? - net_read_timeout : THDVAR(thd, net_read_timeout)); - DBUG_RETURN(net_read_timeout); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, net_read_timeout) /* - -1 :use table parameter + -1 :fallback to default 0-:seconds of timeout */ static MYSQL_THDVAR_INT( @@ -1291,25 +1111,16 @@ static MYSQL_THDVAR_INT( "Wait timeout of sending data to remote server", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 600, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -int spider_param_net_write_timeout( - THD *thd, - int net_write_timeout -) { - DBUG_ENTER("spider_param_net_write_timeout"); - if (thd) - DBUG_RETURN(THDVAR(thd, net_write_timeout) == -1 ? - net_write_timeout : THDVAR(thd, net_write_timeout)); - DBUG_RETURN(net_write_timeout); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, net_write_timeout) /* - -1 :use table parameter + -1 :fallback to default 0 :It acquires it collectively. 1 :Acquisition one by one.If it discontinues once, and it will need it later, it retrieves it again when there is interrupt on the way. @@ -1322,23 +1133,16 @@ static MYSQL_THDVAR_INT( "The retrieval result from a remote server is acquired by acquisition one by one", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 3, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_quick_mode( - THD *thd, - int quick_mode -) { - DBUG_ENTER("spider_param_quick_mode"); - DBUG_RETURN(THDVAR(thd, quick_mode) < 0 ? - quick_mode : THDVAR(thd, quick_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, quick_mode) /* - -1 :use table parameter + -1 :fallback to default 0-:number of records */ static MYSQL_THDVAR_LONGLONG( @@ -1347,23 +1151,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of records in a page when acquisition one by one", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1024, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_quick_page_size( - THD *thd, - longlong quick_page_size -) { - DBUG_ENTER("spider_param_quick_page_size"); - DBUG_RETURN(THDVAR(thd, quick_page_size) < 0 ? - quick_page_size : THDVAR(thd, quick_page_size)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, quick_page_size) /* - -1 :use table parameter + -1 :fallback to default 0-:the limitation of memory size */ static MYSQL_THDVAR_LONGLONG( @@ -1372,23 +1169,16 @@ static MYSQL_THDVAR_LONGLONG( "The limitation of memory size in a page when acquisition one by one", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 10485760, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_quick_page_byte( - THD *thd, - longlong quick_page_byte -) { - DBUG_ENTER("spider_param_quick_page_byte"); - DBUG_RETURN(THDVAR(thd, quick_page_byte) < 0 ? - quick_page_byte : THDVAR(thd, quick_page_byte)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, quick_page_byte) /* - -1 :use table parameter + -1 :fallback to default 0 :It doesn't use low memory mode. 1 :It uses low memory mode. */ @@ -1398,23 +1188,16 @@ static MYSQL_THDVAR_INT( "Use low memory mode when SQL(SELECT) internally issued to a remote server is executed and get a result list", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_low_mem_read( - THD *thd, - int low_mem_read -) { - DBUG_ENTER("spider_param_low_mem_read"); - DBUG_RETURN(THDVAR(thd, low_mem_read) < 0 ? - low_mem_read : THDVAR(thd, low_mem_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, low_mem_read) /* - -1 :use table parameter + -1 :fallback to default 0 :Use index columns if select statement can solve by using index, otherwise use all columns. 1 :Use columns that are judged necessary. @@ -1425,24 +1208,17 @@ static MYSQL_THDVAR_INT( "The mode of using columns at select clause", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_select_column_mode( - THD *thd, - int select_column_mode -) { - DBUG_ENTER("spider_param_select_column_mode"); - DBUG_RETURN(THDVAR(thd, select_column_mode) == -1 ? - select_column_mode : THDVAR(thd, select_column_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, select_column_mode) #ifndef WITHOUT_SPIDER_BG_SEARCH /* - -1 :use table parameter + -1 :fallback to default 0 :background search is disabled 1 :background search is used if search with no lock 2 :background search is used if search with no lock or shared lock @@ -1454,23 +1230,16 @@ static MYSQL_THDVAR_INT( "Mode of background search", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_bgs_mode( - THD *thd, - int bgs_mode -) { - DBUG_ENTER("spider_param_bgs_mode"); - DBUG_RETURN(THDVAR(thd, bgs_mode) < 0 ? - bgs_mode : THDVAR(thd, bgs_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bgs_mode) /* - -1 :use table parameter + -1 :fallback to default 0 :records is gotten usually 1-:number of records */ @@ -1480,23 +1249,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of first read records when background search is used", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_bgs_first_read( - THD *thd, - longlong bgs_first_read -) { - DBUG_ENTER("spider_param_bgs_first_read"); - DBUG_RETURN(THDVAR(thd, bgs_first_read) < 0 ? - bgs_first_read : THDVAR(thd, bgs_first_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, bgs_first_read) /* - -1 :use table parameter + -1 :fallback to default 0 :records is gotten usually 1-:number of records */ @@ -1506,24 +1268,17 @@ static MYSQL_THDVAR_LONGLONG( "Number of second read records when background search is used", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 100, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_bgs_second_read( - THD *thd, - longlong bgs_second_read -) { - DBUG_ENTER("spider_param_bgs_second_read"); - DBUG_RETURN(THDVAR(thd, bgs_second_read) < 0 ? - bgs_second_read : THDVAR(thd, bgs_second_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, bgs_second_read) #endif /* - -1 :use table parameter + -1 :fallback to default 0 :records is gotten usually 1-:number of records */ @@ -1533,23 +1288,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of first read records", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_first_read( - THD *thd, - longlong first_read -) { - DBUG_ENTER("spider_param_first_read"); - DBUG_RETURN(THDVAR(thd, first_read) < 0 ? - first_read : THDVAR(thd, first_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, first_read) /* - -1 :use table parameter + -1 :fallback to default 0 :records is gotten usually 1-:number of records */ @@ -1559,23 +1307,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of second read records", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_second_read( - THD *thd, - longlong second_read -) { - DBUG_ENTER("spider_param_second_read"); - DBUG_RETURN(THDVAR(thd, second_read) < 0 ? - second_read : THDVAR(thd, second_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, second_read) /* - -1 :use table parameter + -1 :fallback to default 0 :always get the newest information 1-:interval */ @@ -1585,23 +1326,16 @@ static MYSQL_THDVAR_INT( "Interval of cardinality confirmation.(second)", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 51, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -double spider_param_crd_interval( - THD *thd, - double crd_interval -) { - DBUG_ENTER("spider_param_crd_interval"); - DBUG_RETURN(THDVAR(thd, crd_interval) == -1 ? - crd_interval : THDVAR(thd, crd_interval)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(double, crd_interval) /* - -1 :use table parameter + -1 :fallback to default 0 :use table parameter 1 :use show command 2 :use information schema @@ -1613,24 +1347,17 @@ static MYSQL_THDVAR_INT( "Mode of cardinality confirmation.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_crd_mode( - THD *thd, - int crd_mode -) { - DBUG_ENTER("spider_param_crd_mode"); - DBUG_RETURN(THDVAR(thd, crd_mode) <= 0 ? - crd_mode : THDVAR(thd, crd_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, crd_mode) #ifdef WITH_PARTITION_STORAGE_ENGINE /* - -1 :use table parameter + -1 :fallback to default 0 :No synchronization. 1 :Cardinality is synchronized when opening a table. Then no synchronization. @@ -1642,24 +1369,17 @@ static MYSQL_THDVAR_INT( "Cardinality synchronization in partitioned table.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_crd_sync( - THD *thd, - int crd_sync -) { - DBUG_ENTER("spider_param_crd_sync"); - DBUG_RETURN(THDVAR(thd, crd_sync) == -1 ? - crd_sync : THDVAR(thd, crd_sync)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, crd_sync) #endif /* - -1 :use table parameter + -1 :fallback to default 0 :The crd_weight is used as a fixed value. 1 :The crd_weight is used as an addition value. 2 :The crd_weight is used as a multiplication value. @@ -1670,23 +1390,16 @@ static MYSQL_THDVAR_INT( "Type of cardinality calculation.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_crd_type( - THD *thd, - int crd_type -) { - DBUG_ENTER("spider_param_crd_type"); - DBUG_RETURN(THDVAR(thd, crd_type) == -1 ? - crd_type : THDVAR(thd, crd_type)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, crd_type) /* - -1 :use table parameter + -1 :fallback to default 0-:weight */ static MYSQL_THDVAR_INT( @@ -1695,24 +1408,17 @@ static MYSQL_THDVAR_INT( "Weight coefficient to calculate effectiveness of index from cardinality of column.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -double spider_param_crd_weight( - THD *thd, - double crd_weight -) { - DBUG_ENTER("spider_param_crd_weight"); - DBUG_RETURN(THDVAR(thd, crd_weight) == -1 ? - crd_weight : THDVAR(thd, crd_weight)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(double, crd_weight) #ifndef WITHOUT_SPIDER_BG_SEARCH /* - -1 :use table parameter + -1 :fallback to default 0 :Background confirmation is disabled 1 :Background confirmation is enabled (create thread per table/partition) 2 :Background confirmation is enabled (use static threads) @@ -1723,24 +1429,17 @@ static MYSQL_THDVAR_INT( "Mode of cardinality confirmation at background.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_crd_bg_mode( - THD *thd, - int crd_bg_mode -) { - DBUG_ENTER("spider_param_crd_bg_mode"); - DBUG_RETURN(THDVAR(thd, crd_bg_mode) == -1 ? - crd_bg_mode : THDVAR(thd, crd_bg_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, crd_bg_mode) #endif /* - -1 :use table parameter + -1 :fallback to default 0 :always get the newest information 1-:interval */ @@ -1750,23 +1449,16 @@ static MYSQL_THDVAR_INT( "Interval of table state confirmation.(second)", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 10, /* def */ -1, /* min */ 2147483647, /* max */ 0 /* blk */ ); -double spider_param_sts_interval( - THD *thd, - double sts_interval -) { - DBUG_ENTER("spider_param_sts_interval"); - DBUG_RETURN(THDVAR(thd, sts_interval) == -1 ? - sts_interval : THDVAR(thd, sts_interval)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(double, sts_interval) /* - -1 :use table parameter + -1 :fallback to default 0 :use table parameter 1 :use show command 2 :use information schema @@ -1777,24 +1469,17 @@ static MYSQL_THDVAR_INT( "Mode of table state confirmation.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_sts_mode( - THD *thd, - int sts_mode -) { - DBUG_ENTER("spider_param_sts_mode"); - DBUG_RETURN(THDVAR(thd, sts_mode) <= 0 ? - sts_mode : THDVAR(thd, sts_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, sts_mode) #ifdef WITH_PARTITION_STORAGE_ENGINE /* - -1 :use table parameter + -1 :fallback to default 0 :No synchronization. 1 :Table state is synchronized when opening a table. Then no synchronization. @@ -1806,25 +1491,18 @@ static MYSQL_THDVAR_INT( "Table state synchronization in partitioned table.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_sts_sync( - THD *thd, - int sts_sync -) { - DBUG_ENTER("spider_param_sts_sync"); - DBUG_RETURN(THDVAR(thd, sts_sync) == -1 ? - sts_sync : THDVAR(thd, sts_sync)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, sts_sync) #endif #ifndef WITHOUT_SPIDER_BG_SEARCH /* - -1 :use table parameter + -1 :fallback to default 0 :Background confirmation is disabled 1 :Background confirmation is enabled (create thread per table/partition) 2 :Background confirmation is enabled (use static threads) @@ -1835,20 +1513,13 @@ static MYSQL_THDVAR_INT( "Mode of table state confirmation at background.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 2, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_sts_bg_mode( - THD *thd, - int sts_bg_mode -) { - DBUG_ENTER("spider_param_sts_bg_mode"); - DBUG_RETURN(THDVAR(thd, sts_bg_mode) == -1 ? - sts_bg_mode : THDVAR(thd, sts_bg_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, sts_bg_mode) #endif /* @@ -1867,12 +1538,7 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -double spider_param_ping_interval_at_trx_start( - THD *thd -) { - DBUG_ENTER("spider_param_ping_interval_at_trx_start"); - DBUG_RETURN(THDVAR(thd, ping_interval_at_trx_start)); -} +SPIDER_THDVAR_VALUE_FUNC(double, ping_interval_at_trx_start) #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) /* @@ -1900,7 +1566,7 @@ double spider_param_hs_ping_interval( #endif /* - -1 :use table parameter + -1 :fallback to default 0 :normal mode 1 :quick mode 2 :set 0 value @@ -1911,20 +1577,13 @@ static MYSQL_THDVAR_INT( "Mode of auto increment.", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_auto_increment_mode( - THD *thd, - int auto_increment_mode -) { - DBUG_ENTER("spider_param_auto_increment_mode"); - DBUG_RETURN(THDVAR(thd, auto_increment_mode) == -1 ? - auto_increment_mode : THDVAR(thd, auto_increment_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, auto_increment_mode) /* FALSE: off @@ -1939,12 +1598,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_same_server_link( - THD *thd -) { - DBUG_ENTER("spider_param_same_server_link"); - DBUG_RETURN(THDVAR(thd, same_server_link)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, same_server_link) /* FALSE: transmits @@ -1960,15 +1614,10 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -bool spider_param_local_lock_table( - THD *thd -) { - DBUG_ENTER("spider_param_local_lock_table"); - DBUG_RETURN(THDVAR(thd, local_lock_table)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, local_lock_table) /* - -1 :use table parameter + -1 :fallback to default 0 :don't transmit 1 :transmits */ @@ -1984,17 +1633,10 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -int spider_param_use_pushdown_udf( - THD *thd, - int use_pushdown_udf -) { - DBUG_ENTER("spider_param_use_pushdown_udf"); - DBUG_RETURN(THDVAR(thd, use_pushdown_udf) == -1 ? - use_pushdown_udf : THDVAR(thd, use_pushdown_udf)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, use_pushdown_udf) /* - -1 :use table parameter + -1 :fallback to default 0 :duplicate check on local server 1 :avoid duplicate check on local server */ @@ -2004,20 +1646,13 @@ static MYSQL_THDVAR_INT( "Execute \"REPLACE\" and \"INSERT IGNORE\" on remote server and avoid duplicate check on local server", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_direct_dup_insert( - THD *thd, - int direct_dup_insert -) { - DBUG_ENTER("spider_param_direct_dup_insert"); - DBUG_RETURN(THDVAR(thd, direct_dup_insert) < 0 ? - direct_dup_insert : THDVAR(thd, direct_dup_insert)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, direct_dup_insert) static uint spider_udf_table_lock_mutex_count; /* @@ -2036,11 +1671,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_udf_table_lock_mutex_count() -{ - DBUG_ENTER("spider_param_udf_table_lock_mutex_count"); - DBUG_RETURN(spider_udf_table_lock_mutex_count); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, udf_table_lock_mutex_count) static uint spider_udf_table_mon_mutex_count; /* @@ -2059,11 +1690,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_udf_table_mon_mutex_count() -{ - DBUG_ENTER("spider_param_udf_table_mon_mutex_count"); - DBUG_RETURN(spider_udf_table_mon_mutex_count); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, udf_table_mon_mutex_count) /* 1-:number of rows @@ -2074,23 +1701,16 @@ static MYSQL_THDVAR_LONGLONG( "Number of rows for bulk inserting", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 3000, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_udf_ds_bulk_insert_rows( - THD *thd, - longlong udf_ds_bulk_insert_rows -) { - DBUG_ENTER("spider_param_udf_ds_bulk_insert_rows"); - DBUG_RETURN(THDVAR(thd, udf_ds_bulk_insert_rows) <= 0 ? - udf_ds_bulk_insert_rows : THDVAR(thd, udf_ds_bulk_insert_rows)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, udf_ds_bulk_insert_rows) /* - -1 :use table parameter + -1 :fallback to default 0 :drop records 1 :insert last table 2 :insert first table and loop again @@ -2101,20 +1721,13 @@ static MYSQL_THDVAR_INT( "Table loop mode if the number of tables in table list are less than the number of result sets", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_udf_ds_table_loop_mode( - THD *thd, - int udf_ds_table_loop_mode -) { - DBUG_ENTER("spider_param_udf_ds_table_loop_mode"); - DBUG_RETURN(THDVAR(thd, udf_ds_table_loop_mode) == -1 ? - udf_ds_table_loop_mode : THDVAR(thd, udf_ds_table_loop_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, udf_ds_table_loop_mode) static char *spider_remote_access_charset; /* @@ -2155,11 +1768,7 @@ static MYSQL_SYSVAR_STR( #endif #endif -char *spider_param_remote_access_charset() -{ - DBUG_ENTER("spider_param_remote_access_charset"); - DBUG_RETURN(spider_remote_access_charset); -} +SPIDER_SYSVAR_VALUE_FUNC(char*, remote_access_charset) static int spider_remote_autocommit; /* @@ -2180,11 +1789,7 @@ static MYSQL_SYSVAR_INT( 0 ); -int spider_param_remote_autocommit() -{ - DBUG_ENTER("spider_param_remote_autocommit"); - DBUG_RETURN(spider_remote_autocommit); -} +SPIDER_SYSVAR_VALUE_FUNC(int, remote_autocommit) static char *spider_remote_time_zone; /* @@ -2225,11 +1830,7 @@ static MYSQL_SYSVAR_STR( #endif #endif -char *spider_param_remote_time_zone() -{ - DBUG_ENTER("spider_param_remote_time_zone"); - DBUG_RETURN(spider_remote_time_zone); -} +SPIDER_SYSVAR_VALUE_FUNC(char *, remote_time_zone) static int spider_remote_sql_log_off; /* @@ -2250,11 +1851,7 @@ static MYSQL_SYSVAR_INT( 0 ); -int spider_param_remote_sql_log_off() -{ - DBUG_ENTER("spider_param_remote_sql_log_off"); - DBUG_RETURN(spider_remote_sql_log_off); -} +SPIDER_SYSVAR_VALUE_FUNC(int, remote_sql_log_off) static int spider_remote_trx_isolation; /* @@ -2277,11 +1874,7 @@ static MYSQL_SYSVAR_INT( 0 ); -int spider_param_remote_trx_isolation() -{ - DBUG_ENTER("spider_param_remote_trx_isolation"); - DBUG_RETURN(spider_remote_trx_isolation); -} +SPIDER_SYSVAR_VALUE_FUNC(int, remote_trx_isolation) static char *spider_remote_default_database; /* @@ -2322,11 +1915,7 @@ static MYSQL_SYSVAR_STR( #endif #endif -char *spider_param_remote_default_database() -{ - DBUG_ENTER("spider_param_remote_default_database"); - DBUG_RETURN(spider_remote_default_database); -} +SPIDER_SYSVAR_VALUE_FUNC(char *, remote_default_database) /* 0-:connect retry interval (micro second) @@ -2416,12 +2005,11 @@ char *spider_param_bka_engine( char *bka_engine ) { DBUG_ENTER("spider_param_bka_engine"); - DBUG_RETURN(THDVAR(thd, bka_engine) ? - THDVAR(thd, bka_engine) : bka_engine); + DBUG_RETURN(bka_engine ? bka_engine : THDVAR(thd, bka_engine)); } /* - -1 :use table parameter + -1 :fallback to default 0 :use union all 1 :use temporary table */ @@ -2431,24 +2019,17 @@ static MYSQL_THDVAR_INT( "Mode of BKA for Spider", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 2, /* max */ 0 /* blk */ ); -int spider_param_bka_mode( - THD *thd, - int bka_mode -) { - DBUG_ENTER("spider_param_bka_mode"); - DBUG_RETURN(THDVAR(thd, bka_mode) == -1 ? - bka_mode : THDVAR(thd, bka_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bka_mode) static int spider_udf_ct_bulk_insert_interval; /* - -1 : The UDF parameter is adopted. + -1 : Fallback to default. 0 or more : Milliseconds. */ static MYSQL_SYSVAR_INT( @@ -2458,23 +2039,17 @@ static MYSQL_SYSVAR_INT( "The interval time between bulk insert and next bulk insert at coping", NULL, NULL, - -1, + 10, -1, 2147483647, 0 ); -int spider_param_udf_ct_bulk_insert_interval( - int udf_ct_bulk_insert_interval -) { - DBUG_ENTER("spider_param_udf_ct_bulk_insert_interval"); - DBUG_RETURN(spider_udf_ct_bulk_insert_interval < 0 ? - udf_ct_bulk_insert_interval : spider_udf_ct_bulk_insert_interval); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, udf_ct_bulk_insert_interval) static longlong spider_udf_ct_bulk_insert_rows; /* - -1,0 : The UDF parameter is adopted. + -1,0 : Fallback to default. 1 or more : Number of rows. */ static MYSQL_SYSVAR_LONGLONG( @@ -2484,19 +2059,13 @@ static MYSQL_SYSVAR_LONGLONG( "The number of rows inserted with bulk insert of one time at coping", NULL, NULL, - -1, + 100, -1, 9223372036854775807LL, 0 ); -longlong spider_param_udf_ct_bulk_insert_rows( - longlong udf_ct_bulk_insert_rows -) { - DBUG_ENTER("spider_param_udf_ct_bulk_insert_rows"); - DBUG_RETURN(spider_udf_ct_bulk_insert_rows <= 0 ? - udf_ct_bulk_insert_rows : spider_udf_ct_bulk_insert_rows); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(longlong, udf_ct_bulk_insert_rows) #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) /* @@ -2594,7 +2163,7 @@ uint spider_param_hs_w_conn_recycle_strict( } /* - -1 :use table parameter + -1 :fallback to default 0 :not use 1 :use handlersocket */ @@ -2604,23 +2173,16 @@ static MYSQL_THDVAR_INT( "Use handlersocket for reading", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_use_hs_read( - THD *thd, - int use_hs_read -) { - DBUG_ENTER("spider_param_use_hs_read"); - DBUG_RETURN(THDVAR(thd, use_hs_read) == -1 ? - use_hs_read : THDVAR(thd, use_hs_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUN(int, use_hs_read) /* - -1 :use table parameter + -1 :fallback to default 0 :not use 1 :use handlersocket */ @@ -2630,24 +2192,17 @@ static MYSQL_THDVAR_INT( "Use handlersocket for writing", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_use_hs_write( - THD *thd, - int use_hs_write -) { - DBUG_ENTER("spider_param_use_hs_write"); - DBUG_RETURN(THDVAR(thd, use_hs_write) == -1 ? - use_hs_write : THDVAR(thd, use_hs_write)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUN(int, use_hs_write) #endif /* - -1 :use table parameter + -1 :fallback to default 0 :not use 1 :use handler */ @@ -2657,23 +2212,16 @@ static MYSQL_THDVAR_INT( "Use handler for reading", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_use_handler( - THD *thd, - int use_handler -) { - DBUG_ENTER("spider_param_use_handler"); - DBUG_RETURN(THDVAR(thd, use_handler) == -1 ? - use_handler : THDVAR(thd, use_handler)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, use_handler) /* - -1 :use table parameter + -1 :fallback to default 0 :return error if error 1 :return 0 record if error */ @@ -2683,23 +2231,16 @@ static MYSQL_THDVAR_INT( "Read error mode if error", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_error_read_mode( - THD *thd, - int error_read_mode -) { - DBUG_ENTER("spider_param_error_read_mode"); - DBUG_RETURN(THDVAR(thd, error_read_mode) == -1 ? - error_read_mode : THDVAR(thd, error_read_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, error_read_mode) /* - -1 :use table parameter + -1 :fallback to default 0 :return error if error 1 :return 0 record if error */ @@ -2709,23 +2250,16 @@ static MYSQL_THDVAR_INT( "Write error mode if error", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_error_write_mode( - THD *thd, - int error_write_mode -) { - DBUG_ENTER("spider_param_error_write_mode"); - DBUG_RETURN(THDVAR(thd, error_write_mode) == -1 ? - error_write_mode : THDVAR(thd, error_write_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, error_write_mode) /* - -1 :use table parameter + -1 :fallback to default 0 :not skip 1 :skip */ @@ -2735,23 +2269,16 @@ static MYSQL_THDVAR_INT( "Skip generating internal default condition", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_skip_default_condition( - THD *thd, - int skip_default_condition -) { - DBUG_ENTER("spider_param_skip_default_condition"); - DBUG_RETURN(THDVAR(thd, skip_default_condition) == -1 ? - skip_default_condition : THDVAR(thd, skip_default_condition)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, skip_default_condition) /* - -1 :use table parameter + -1 :fallback to default 0 :not skip 1 :skip parallel search if query is not SELECT statement 2 :skip parallel search if query has SQL_NO_CACHE @@ -2763,23 +2290,16 @@ static MYSQL_THDVAR_INT( "Skip parallel search by specific conditions", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 3, /* max */ 0 /* blk */ ); -int spider_param_skip_parallel_search( - THD *thd, - int skip_parallel_search -) { - DBUG_ENTER("spider_param_skip_parallel_search"); - DBUG_RETURN(THDVAR(thd, skip_parallel_search) == -1 ? - skip_parallel_search : THDVAR(thd, skip_parallel_search)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, skip_parallel_search) /* - -1 :use table parameter + -1 :fallback to default 0 :not send directly 1-:send directly */ @@ -2789,23 +2309,16 @@ static MYSQL_THDVAR_LONGLONG( "Send 'ORDER BY' and 'LIMIT' to remote server directly", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 9223372036854775807LL, /* def */ -1, /* min */ 9223372036854775807LL, /* max */ 0 /* blk */ ); -longlong spider_param_direct_order_limit( - THD *thd, - longlong direct_order_limit -) { - DBUG_ENTER("spider_param_direct_order_limit"); - DBUG_RETURN(THDVAR(thd, direct_order_limit) == -1 ? - direct_order_limit : THDVAR(thd, direct_order_limit)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(longlong, direct_order_limit) /* - -1 :use table parameter + -1 :fallback to default 0 :writable 1 :read only */ @@ -2815,25 +2328,18 @@ static MYSQL_THDVAR_INT( "Read only", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_read_only_mode( - THD *thd, - int read_only_mode -) { - DBUG_ENTER("spider_param_read_only_mode"); - DBUG_RETURN(THDVAR(thd, read_only_mode) == -1 ? - read_only_mode : THDVAR(thd, read_only_mode)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, read_only_mode) #ifdef HA_CAN_BULK_ACCESS static int spider_bulk_access_free; /* - -1 :use table parameter + -1 :fallback to default 0 :in reset 1 :in close */ @@ -2844,25 +2350,19 @@ static MYSQL_SYSVAR_INT( "Free mode of bulk access resources", NULL, NULL, - -1, + 0, -1, 1, 0 ); -int spider_param_bulk_access_free( - int bulk_access_free -) { - DBUG_ENTER("spider_param_bulk_access_free"); - DBUG_RETURN(spider_bulk_access_free == -1 ? - bulk_access_free : spider_bulk_access_free); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUN(int, bulk_access_free) #endif #if MYSQL_VERSION_ID < 50500 #else /* - -1 :use UDF parameter + -1 :fallback to default 0 :can not use 1 :can use */ @@ -2872,20 +2372,13 @@ static MYSQL_THDVAR_INT( "Use real table for temporary table list", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_udf_ds_use_real_table( - THD *thd, - int udf_ds_use_real_table -) { - DBUG_ENTER("spider_param_udf_ds_use_real_table"); - DBUG_RETURN(THDVAR(thd, udf_ds_use_real_table) == -1 ? - udf_ds_use_real_table : THDVAR(thd, udf_ds_use_real_table)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, udf_ds_use_real_table) #endif static my_bool spider_general_log; @@ -2899,11 +2392,7 @@ static MYSQL_SYSVAR_BOOL( FALSE ); -my_bool spider_param_general_log() -{ - DBUG_ENTER("spider_param_general_log"); - DBUG_RETURN(spider_general_log); -} +SPIDER_SYSVAR_VALUE_FUNC(my_bool, general_log) /* FALSE: no pushdown hints @@ -2918,12 +2407,7 @@ static MYSQL_THDVAR_BOOL( FALSE /* def */ ); -my_bool spider_param_index_hint_pushdown( - THD *thd -) { - DBUG_ENTER("spider_param_index_hint_pushdown"); - DBUG_RETURN(THDVAR(thd, index_hint_pushdown)); -} +SPIDER_THDVAR_VALUE_FUNC(my_bool, index_hint_pushdown) static uint spider_max_connections; static MYSQL_SYSVAR_UINT( @@ -2939,11 +2423,7 @@ static MYSQL_SYSVAR_UINT( 0 /* blk */ ); -uint spider_param_max_connections() -{ - DBUG_ENTER("spider_param_max_connections"); - DBUG_RETURN(spider_max_connections); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, max_connections) static uint spider_conn_wait_timeout; static MYSQL_SYSVAR_UINT( @@ -2959,11 +2439,7 @@ static MYSQL_SYSVAR_UINT( 0 /* blk */ ); -uint spider_param_conn_wait_timeout() -{ - DBUG_ENTER("spider_param_conn_wait_timeout"); - DBUG_RETURN(spider_conn_wait_timeout); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, conn_wait_timeout) static uint spider_log_result_errors; /* @@ -2986,11 +2462,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_log_result_errors() -{ - DBUG_ENTER("spider_param_log_result_errors"); - DBUG_RETURN(spider_log_result_errors); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, log_result_errors) static uint spider_log_result_error_with_sql; /* @@ -3012,11 +2484,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_log_result_error_with_sql() -{ - DBUG_ENTER("spider_param_log_result_error_with_sql"); - DBUG_RETURN(spider_log_result_error_with_sql); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, log_result_error_with_sql) static char *spider_version = (char *) SPIDER_DETAIL_VERSION; static MYSQL_SYSVAR_STR( @@ -3045,15 +2513,10 @@ static MYSQL_THDVAR_UINT( 0 /* blk */ ); -uint spider_param_internal_xa_id_type( - THD *thd -) { - DBUG_ENTER("spider_param_internal_xa_id_type"); - DBUG_RETURN(THDVAR(thd, internal_xa_id_type)); -} +SPIDER_THDVAR_VALUE_FUNC(uint, internal_xa_id_type) /* - -1 :use table parameter + -1 :fallback to default 0 :OFF 1 :automatic channel 2-63 :use custom channel @@ -3064,20 +2527,13 @@ static MYSQL_THDVAR_INT( "Read casually if it is possible", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 63, /* max */ 0 /* blk */ ); -int spider_param_casual_read( - THD *thd, - int casual_read -) { - DBUG_ENTER("spider_param_casual_read"); - DBUG_RETURN(THDVAR(thd, casual_read) == -1 ? - casual_read : THDVAR(thd, casual_read)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, casual_read) static my_bool spider_dry_access; static MYSQL_SYSVAR_BOOL( @@ -3090,14 +2546,10 @@ static MYSQL_SYSVAR_BOOL( FALSE ); -my_bool spider_param_dry_access() -{ - DBUG_ENTER("spider_param_dry_access"); - DBUG_RETURN(spider_dry_access); -} +SPIDER_SYSVAR_VALUE_FUNC(my_bool, dry_access) /* - -1 :use table parameter + -1 :fallback to default 0 :fast 1 :correct delete row number */ @@ -3107,23 +2559,16 @@ static MYSQL_THDVAR_INT( "The type of delete_all_rows", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 1, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_delete_all_rows_type( - THD *thd, - int delete_all_rows_type -) { - DBUG_ENTER("spider_param_delete_all_rows_type"); - DBUG_RETURN(THDVAR(thd, delete_all_rows_type) == -1 ? - delete_all_rows_type : THDVAR(thd, delete_all_rows_type)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, delete_all_rows_type) /* - -1 :use table parameter + -1 :fallback to default 0 :compact 1 :add original table name */ @@ -3133,23 +2578,16 @@ static MYSQL_THDVAR_INT( "The type of temporary table name for bka", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ ); -int spider_param_bka_table_name_type( - THD *thd, - int bka_table_name_type -) { - DBUG_ENTER("spider_param_bka_table_name_type"); - DBUG_RETURN(THDVAR(thd, bka_table_name_type) == -1 ? - bka_table_name_type : THDVAR(thd, bka_table_name_type)); -} +SPIDER_THDVAR_OVERRIDE_VALUE_FUNC(int, bka_table_name_type) /* - -1 :use table parameter + -1 :fallback to default 0 :off 1 :on */ @@ -3165,16 +2603,11 @@ static MYSQL_THDVAR_INT( 0 /* blk */ ); -int spider_param_use_cond_other_than_pk_for_update( - THD *thd -) { - DBUG_ENTER("spider_param_reset_sql_alloc"); - DBUG_RETURN(THDVAR(thd, use_cond_other_than_pk_for_update)); -} +SPIDER_THDVAR_VALUE_FUNC(int, use_cond_other_than_pk_for_update) static int spider_store_last_sts; /* - -1 : use table parameter + -1 : fallback to default 0 : do not store 1 : do store */ @@ -3185,23 +2618,17 @@ static MYSQL_SYSVAR_INT( "Store last sts result into system table", NULL, NULL, - -1, + 1, -1, 1, 0 ); -int spider_param_store_last_sts( - int store_last_sts -) { - DBUG_ENTER("spider_param_store_last_sts"); - DBUG_RETURN(spider_store_last_sts == -1 ? - store_last_sts : spider_store_last_sts); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, store_last_sts) static int spider_store_last_crd; /* - -1 : use table parameter + -1 : fallback to default 0 : do not store 1 : do store */ @@ -3212,23 +2639,17 @@ static MYSQL_SYSVAR_INT( "Store last crd result into system table", NULL, NULL, - -1, + 1, -1, 1, 0 ); -int spider_param_store_last_crd( - int store_last_crd -) { - DBUG_ENTER("spider_param_store_last_crd"); - DBUG_RETURN(spider_store_last_crd == -1 ? - store_last_crd : spider_store_last_crd); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, store_last_crd) static int spider_load_sts_at_startup; /* - -1 : use table parameter + -1 : fallback to default 0 : do not load 1 : do load */ @@ -3239,23 +2660,17 @@ static MYSQL_SYSVAR_INT( "Load sts from system table at startup", NULL, NULL, - -1, + 1, -1, 1, 0 ); -int spider_param_load_sts_at_startup( - int load_sts_at_startup -) { - DBUG_ENTER("spider_param_load_sts_at_startup"); - DBUG_RETURN(spider_load_sts_at_startup == -1 ? - load_sts_at_startup : spider_load_sts_at_startup); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, load_sts_at_startup) static int spider_load_crd_at_startup; /* - -1 : use table parameter + -1 : fallback to default 0 : do not load 1 : do load */ @@ -3266,19 +2681,13 @@ static MYSQL_SYSVAR_INT( "Load crd from system table at startup", NULL, NULL, - -1, + 1, -1, 1, 0 ); -int spider_param_load_crd_at_startup( - int load_crd_at_startup -) { - DBUG_ENTER("spider_param_load_crd_at_startup"); - DBUG_RETURN(spider_load_crd_at_startup == -1 ? - load_crd_at_startup : spider_load_crd_at_startup); -} +SPIDER_SYSVAR_OVERRIDE_VALUE_FUNC(int, load_crd_at_startup) #ifndef WITHOUT_SPIDER_BG_SEARCH static uint spider_table_sts_thread_count; @@ -3298,11 +2707,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_table_sts_thread_count() -{ - DBUG_ENTER("spider_param_table_sts_thread_count"); - DBUG_RETURN(spider_table_sts_thread_count); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, table_sts_thread_count) static uint spider_table_crd_thread_count; /* @@ -3321,11 +2726,7 @@ static MYSQL_SYSVAR_UINT( 0 ); -uint spider_param_table_crd_thread_count() -{ - DBUG_ENTER("spider_param_table_crd_thread_count"); - DBUG_RETURN(spider_table_crd_thread_count); -} +SPIDER_SYSVAR_VALUE_FUNC(uint, table_crd_thread_count) #endif static int spider_slave_trx_isolation; @@ -3349,11 +2750,7 @@ static MYSQL_SYSVAR_INT( 0 /* blk */ ); -int spider_param_slave_trx_isolation() -{ - DBUG_ENTER("spider_param_slave_trx_isolation"); - DBUG_RETURN(spider_slave_trx_isolation); -} +SPIDER_SYSVAR_VALUE_FUNC(int, slave_trx_isolation) /* -1 :not set @@ -3418,12 +2815,7 @@ static MYSQL_THDVAR_BOOL( TRUE /* def */ ); -bool spider_param_sync_sql_mode( - THD *thd -) { - DBUG_ENTER("spider_param_sync_sql_mode"); - DBUG_RETURN(THDVAR(thd, sync_sql_mode)); -} +SPIDER_THDVAR_VALUE_FUNC(bool, sync_sql_mode) static struct st_mysql_storage_engine spider_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION }; diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index 2ff97ce3577..a07df994e46 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -3700,10 +3700,6 @@ int spider_set_connect_info_default( #endif #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) - if (share->use_hs_reads[roop_count] == -1) - share->use_hs_reads[roop_count] = 0; - if (share->use_hs_writes[roop_count] == -1) - share->use_hs_writes[roop_count] = 0; if (share->hs_read_ports[roop_count] == -1) { share->hs_read_ports[roop_count] = 9998; @@ -3735,154 +3731,24 @@ int spider_set_connect_info_default( share->hs_write_to_reads[roop_count] = 1; } #endif - if (share->use_handlers[roop_count] == -1) - share->use_handlers[roop_count] = 0; - if (share->connect_timeouts[roop_count] == -1) - share->connect_timeouts[roop_count] = 6; - if (share->net_read_timeouts[roop_count] == -1) - share->net_read_timeouts[roop_count] = 600; - if (share->net_write_timeouts[roop_count] == -1) - share->net_write_timeouts[roop_count] = 600; if (share->access_balances[roop_count] == -1) share->access_balances[roop_count] = 100; - if (share->bka_table_name_types[roop_count] == -1) - share->bka_table_name_types[roop_count] = 0; } -#ifndef WITHOUT_SPIDER_BG_SEARCH - if (share->sts_bg_mode == -1) - share->sts_bg_mode = 2; -#endif - if (share->sts_interval == -1) - share->sts_interval = 10; - if (share->sts_mode == -1) - share->sts_mode = 1; -#ifdef WITH_PARTITION_STORAGE_ENGINE - if (share->sts_sync == -1) - share->sts_sync = 0; -#endif - if (share->store_last_sts == -1) - share->store_last_sts = 1; - if (share->load_sts_at_startup == -1) - share->load_sts_at_startup = 1; -#ifndef WITHOUT_SPIDER_BG_SEARCH - if (share->crd_bg_mode == -1) - share->crd_bg_mode = 2; -#endif - if (share->crd_interval == -1) - share->crd_interval = 51; - if (share->crd_mode == -1) - share->crd_mode = 1; -#ifdef WITH_PARTITION_STORAGE_ENGINE - if (share->crd_sync == -1) - share->crd_sync = 0; -#endif - if (share->store_last_crd == -1) - share->store_last_crd = 1; - if (share->load_crd_at_startup == -1) - share->load_crd_at_startup = 1; - if (share->crd_type == -1) - share->crd_type = 2; - if (share->crd_weight == -1) - share->crd_weight = 2; - if (share->internal_offset == -1) - share->internal_offset = 0; - if (share->internal_limit == -1) - share->internal_limit = 9223372036854775807LL; - if (share->split_read == -1) - share->split_read = 9223372036854775807LL; - if (share->semi_split_read == -1) - share->semi_split_read = 2; - if (share->semi_split_read_limit == -1) - share->semi_split_read_limit = 9223372036854775807LL; - if (share->init_sql_alloc_size == -1) - share->init_sql_alloc_size = 1024; - if (share->reset_sql_alloc == -1) - share->reset_sql_alloc = 1; - if (share->multi_split_read == -1) - share->multi_split_read = 100; - if (share->max_order == -1) - share->max_order = 32767; - if (share->semi_table_lock == -1) - share->semi_table_lock = 0; - if (share->semi_table_lock_conn == -1) - share->semi_table_lock_conn = 1; - if (share->selupd_lock_mode == -1) - share->selupd_lock_mode = 1; if (share->query_cache == -1) share->query_cache = 0; if (share->query_cache_sync == -1) share->query_cache_sync = 0; - if (share->bulk_size == -1) - share->bulk_size = 16000; - if (share->bulk_update_mode == -1) - share->bulk_update_mode = 0; - if (share->bulk_update_size == -1) - share->bulk_update_size = 16000; - if (share->internal_optimize == -1) - share->internal_optimize = 0; - if (share->internal_optimize_local == -1) - share->internal_optimize_local = 0; if (share->scan_rate == -1) share->scan_rate = 1; if (share->read_rate == -1) share->read_rate = 0.0002; if (share->priority == -1) share->priority = 1000000; - if (share->quick_mode == -1) - share->quick_mode = 3; - if (share->quick_page_size == -1) - share->quick_page_size = 1024; - if (share->quick_page_byte == -1) - share->quick_page_byte = 10485760; - if (share->low_mem_read == -1) - share->low_mem_read = 1; if (share->table_count_mode == -1) share->table_count_mode = 0; - if (share->select_column_mode == -1) - share->select_column_mode = 1; -#ifndef WITHOUT_SPIDER_BG_SEARCH - if (share->bgs_mode == -1) - share->bgs_mode = 0; - if (share->bgs_first_read == -1) - share->bgs_first_read = 2; - if (share->bgs_second_read == -1) - share->bgs_second_read = 100; -#endif - if (share->first_read == -1) - share->first_read = 0; - if (share->second_read == -1) - share->second_read = 0; - if (share->auto_increment_mode == -1) - share->auto_increment_mode = 0; - if (share->use_table_charset == -1) - share->use_table_charset = 1; - if (share->use_pushdown_udf == -1) - share->use_pushdown_udf = 1; - if (share->skip_default_condition == -1) - share->skip_default_condition = 0; - if (share->skip_parallel_search == -1) - share->skip_parallel_search = 0; - if (share->direct_dup_insert == -1) - share->direct_dup_insert = 0; - if (share->direct_order_limit == -1) - share->direct_order_limit = 9223372036854775807LL; - if (share->read_only_mode == -1) - share->read_only_mode = 0; - if (share->error_read_mode == -1) - share->error_read_mode = 0; - if (share->error_write_mode == -1) - share->error_write_mode = 0; if (share->active_link_count == -1) share->active_link_count = share->all_link_count; -#if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) - if (share->hs_result_free_size == -1) - share->hs_result_free_size = 1048576; -#endif -#ifdef HA_CAN_BULK_ACCESS - if (share->bulk_access_free == -1) - share->bulk_access_free = 0; -#endif #ifdef HA_CAN_FORCE_BULK_UPDATE if (share->force_bulk_update == -1) share->force_bulk_update = 0; @@ -3891,18 +3757,6 @@ int spider_set_connect_info_default( if (share->force_bulk_delete == -1) share->force_bulk_delete = 0; #endif - if (share->casual_read == -1) - share->casual_read = 0; - if (share->delete_all_rows_type == -1) - { -#ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS - share->delete_all_rows_type = 1; -#else - share->delete_all_rows_type = 0; -#endif - } - if (share->bka_mode == -1) - share->bka_mode = 1; if (!share->bka_engine) { DBUG_PRINT("info",("spider create default bka_engine")); From b884216be74bad8c79417633fa40d63fce7931cf Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 14 Jul 2023 10:13:17 +0200 Subject: [PATCH 18/43] MDEV-28017 Illegal mix of collations (cp1251_general_ci,IMPLICIT), (latin1_swedish_ci,COERCIBLE), (latin1_swedish_ci,COERCIBLE) for operation 'between' Correct test fix is to disable service connection to create views in connection with correct charset settings. --- mysql-test/main/ctype_recoding.test | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/ctype_recoding.test b/mysql-test/main/ctype_recoding.test index cc58d4aeff9..9b5f74d0b93 100644 --- a/mysql-test/main/ctype_recoding.test +++ b/mysql-test/main/ctype_recoding.test @@ -178,8 +178,7 @@ drop table t1; # # Check more automatic conversion # -# Enable view protocol after fix MDEV-28017 ---disable_view_protocol +--disable_service_connection set names koi8r; create table t1 (c1 char(10) character set cp1251); insert into t1 values ('ß'); @@ -204,7 +203,7 @@ select rpad(c1,3,' #select case c1 when 'ß' then 'ß' when 'ö' then 'ö' else 'c' end from t1; #select export_set(5,c1,'ö'), export_set(5,'ö',c1) from t1; drop table t1; ---enable_view_protocol +--enable_service_connection # # Bug 20695: problem with field default value's character set From 4b3f93063958834e3d55089d23d16ad242430bf8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 10 Jul 2023 11:40:00 +1000 Subject: [PATCH 19/43] MDEV-31336: pam_user_map : not supporting username or groupname containing @ character Add @ to the allowed characters in a username. --- plugin/auth_pam/mapper/pam_user_map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/auth_pam/mapper/pam_user_map.c b/plugin/auth_pam/mapper/pam_user_map.c index fa8d9ae08c1..5dda97a20cd 100644 --- a/plugin/auth_pam/mapper/pam_user_map.c +++ b/plugin/auth_pam/mapper/pam_user_map.c @@ -216,7 +216,7 @@ int pam_sm_authenticate(pam_handle_t *pamh, int flags, } from= s; skip(isalnum(*s) || (*s == '_') || (*s == '.') || (*s == '-') || - (*s == '$') || (*s == '\\') || (*s == '/')); + (*s == '$') || (*s == '\\') || (*s == '/') || (*s == '@')); end_from= s; skip(isspace(*s)); if (end_from == from || *s++ != ':') goto syntax_error; From fbc157ab33bb2b7a239f13f8b64ce5935f0bdee9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 18 Jul 2023 11:59:42 +1000 Subject: [PATCH 20/43] MDEV-31545 GCC 13 -Wdangling-pointer in execute_show_status() The pointer was used deep in the call path. Resolve this by setting the pointer to NULL at the end of the function. Tested with gcc-13.3.1 (fc38) The warning disable 38fe266ea953 can be reverted in 10.6+ on merge. --- sql/sql_parse.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 473b23fd20c..f8039b7df09 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -6522,9 +6522,11 @@ static bool execute_show_status(THD *thd, TABLE_LIST *all_tables) memcpy(&thd->status_var, &old_status_var, offsetof(STATUS_VAR, last_cleared_system_status_var)); mysql_mutex_unlock(&LOCK_status); + thd->initial_status_var= NULL; return res; #ifdef WITH_WSREP wsrep_error_label: /* see WSREP_SYNC_WAIT() macro above */ + thd->initial_status_var= NULL; return true; #endif /* WITH_WSREP */ } From 9e5c1fb5d30f76586f52aa799d4af3b76e1438b9 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 19 Jul 2023 06:13:44 +0400 Subject: [PATCH 21/43] MDEV-23838 Possibly wrong result or Assertion `0' failed in Item_func_round::native_op Change history in the affected code: - Since 10.4.8 (MDEV-20397 and MDEV-23311), functions ROUND(), CEILING(), FLOOR() return a TIME value for a TIME input. - Since 10.4.14 (MDEV-23525), MIN() and MAX() calculate a result for a TIME input using val_native() rather than val_str(). Problem: The patch for MDEV-23525 did not take into account combinations like MIN(ROUND(time)), MAX(FLOOR(time)), etc. MIN() and MAX() with ROUND(time), CEILING(time), FLOOR(time) as an argument call the method val_native() of the undelying classes Item_func_round and Item_func_int_val. However these classes implemented the method val_native() as DBUG_ASSERT(0). Fix: This patch adds a TIME-specific code inside: - Item_func_round::val_native() - Item_func_int_val::val_native() still with DBUG_ASSERT(0) for all other data types, as other data types do not call val_native() of these classes. We'll need a more generic solition eventualy, e.g. turn Item_func_round and Item_func_int_val into Item_handled_func. However, this change would be too risky for 10.4 at this point. --- mysql-test/main/type_time.result | 44 ++++++++++++++++++++++++++++++++ mysql-test/main/type_time.test | 35 +++++++++++++++++++++++++ sql/item_func.cc | 20 +++++++++++++++ sql/item_func.h | 12 ++------- 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/mysql-test/main/type_time.result b/mysql-test/main/type_time.result index 7b40328892d..f23849eb0c2 100644 --- a/mysql-test/main/type_time.result +++ b/mysql-test/main/type_time.result @@ -2446,5 +2446,49 @@ DROP TABLE t1; SET @@time_zone=DEFAULT; SET TIMESTAMP=DEFAULT; # +# MDEV-23838 Possibly wrong result or Assertion `0' failed in Item_func_round::native_op +# +CREATE TABLE t1 (a TIME, b INT); +INSERT INTO t1 VALUES ('07:26:24',NULL),('23:55:04',NULL); +SELECT MAX(ROUND(a, 0)) FROM t1 GROUP BY 1 << b; +MAX(ROUND(a, 0)) +23:55:04 +DROP TABLE t1; +CREATE TABLE t1 (a TIME(6), b INT); +INSERT INTO t1 VALUES ('07:26:24.12',NULL),('23:55:04.34',NULL); +SELECT MAX(ROUND(a, 2)) FROM t1 GROUP BY 1 << b; +MAX(ROUND(a, 2)) +23:55:04.34 +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (a TIME); +INSERT INTO t1 VALUES (0),(0); +SELECT MAX(ROUND (a,a)) FROM t1 GROUP BY a; +MAX(ROUND (a,a)) +00:00:00 +DROP TABLE t1; +SET sql_mode=DEFAULT; +CREATE TABLE t1 (a TIME, b INT); +INSERT INTO t1 VALUES ('07:26:24',1),('23:55:04',1); +SELECT MIN(CEILING(a)), MAX(CEILING(a)) FROM t1 GROUP BY b; +MIN(CEILING(a)) MAX(CEILING(a)) +07:26:24 23:55:04 +SELECT MIN(FLOOR(a)), MAX(FLOOR(a)) FROM t1 GROUP BY b; +MIN(FLOOR(a)) MAX(FLOOR(a)) +07:26:24 23:55:04 +DROP TABLE t1; +CREATE TABLE t1 (a TIME(6), b INT); +INSERT INTO t1 VALUES ('00:00:00.5',1),('00:01:00.5',1); +INSERT INTO t1 VALUES ('-00:00:00.5',2),('-00:01:00.5',2); +SELECT MIN(CEILING(a)), MAX(CEILING(a)) FROM t1 GROUP BY b; +MIN(CEILING(a)) MAX(CEILING(a)) +00:00:01 00:01:01 +-00:01:00 00:00:00 +SELECT MIN(FLOOR(a)), MAX(FLOOR(a)) FROM t1 GROUP BY b; +MIN(FLOOR(a)) MAX(FLOOR(a)) +00:00:00 00:01:00 +-00:01:01 -00:00:01 +DROP TABLE t1; +# # End of 10.4 tests # diff --git a/mysql-test/main/type_time.test b/mysql-test/main/type_time.test index 9ed5c3a73dc..a75b278b1de 100644 --- a/mysql-test/main/type_time.test +++ b/mysql-test/main/type_time.test @@ -1585,6 +1585,41 @@ SET @@time_zone=DEFAULT; SET TIMESTAMP=DEFAULT; +--echo # +--echo # MDEV-23838 Possibly wrong result or Assertion `0' failed in Item_func_round::native_op +--echo # + +CREATE TABLE t1 (a TIME, b INT); +INSERT INTO t1 VALUES ('07:26:24',NULL),('23:55:04',NULL); +SELECT MAX(ROUND(a, 0)) FROM t1 GROUP BY 1 << b; +DROP TABLE t1; + +CREATE TABLE t1 (a TIME(6), b INT); +INSERT INTO t1 VALUES ('07:26:24.12',NULL),('23:55:04.34',NULL); +SELECT MAX(ROUND(a, 2)) FROM t1 GROUP BY 1 << b; +DROP TABLE t1; + +SET sql_mode=''; +CREATE TABLE t1 (a TIME); +INSERT INTO t1 VALUES (0),(0); +SELECT MAX(ROUND (a,a)) FROM t1 GROUP BY a; +DROP TABLE t1; +SET sql_mode=DEFAULT; + +CREATE TABLE t1 (a TIME, b INT); +INSERT INTO t1 VALUES ('07:26:24',1),('23:55:04',1); +SELECT MIN(CEILING(a)), MAX(CEILING(a)) FROM t1 GROUP BY b; +SELECT MIN(FLOOR(a)), MAX(FLOOR(a)) FROM t1 GROUP BY b; +DROP TABLE t1; + +CREATE TABLE t1 (a TIME(6), b INT); +INSERT INTO t1 VALUES ('00:00:00.5',1),('00:01:00.5',1); +INSERT INTO t1 VALUES ('-00:00:00.5',2),('-00:01:00.5',2); +SELECT MIN(CEILING(a)), MAX(CEILING(a)) FROM t1 GROUP BY b; +SELECT MIN(FLOOR(a)), MAX(FLOOR(a)) FROM t1 GROUP BY b; +DROP TABLE t1; + + --echo # --echo # End of 10.4 tests --echo # diff --git a/sql/item_func.cc b/sql/item_func.cc index 4575990a7a2..fd393f8de3f 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2233,6 +2233,16 @@ bool Item_func_int_val::fix_length_and_dec() } +bool Item_func_int_val::native_op(THD *thd, Native *to) +{ + // TODO: turn Item_func_int_val into Item_handled_func eventually. + if (type_handler()->mysql_timestamp_type() == MYSQL_TIMESTAMP_TIME) + return Time(thd, this).to_native(to, decimals); + DBUG_ASSERT(0); + return true; +} + + longlong Item_func_ceiling::int_op() { switch (args[0]->result_type()) { @@ -2661,6 +2671,16 @@ bool Item_func_round::date_op(THD *thd, MYSQL_TIME *to, date_mode_t fuzzydate) } +bool Item_func_round::native_op(THD *thd, Native *to) +{ + // TODO: turn Item_func_round into Item_handled_func eventually. + if (type_handler()->mysql_timestamp_type() == MYSQL_TIMESTAMP_TIME) + return Time(thd, this).to_native(to, decimals); + DBUG_ASSERT(0); + return true; +} + + void Item_func_rand::seed_random(Item *arg) { /* diff --git a/sql/item_func.h b/sql/item_func.h index 7828078ac7b..76a997c33fb 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1748,11 +1748,7 @@ public: } bool fix_length_and_dec(); String *str_op(String *str) { DBUG_ASSERT(0); return 0; } - bool native_op(THD *thd, Native *to) - { - DBUG_ASSERT(0); - return true; - } + bool native_op(THD *thd, Native *to); }; @@ -1804,11 +1800,7 @@ public: my_decimal *decimal_op(my_decimal *); bool date_op(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate); bool time_op(THD *thd, MYSQL_TIME *ltime); - bool native_op(THD *thd, Native *to) - { - DBUG_ASSERT(0); - return true; - } + bool native_op(THD *thd, Native *to); String *str_op(String *str) { DBUG_ASSERT(0); From 30f3db3cf12d05c5da2c5cd279b6a2b122363277 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 19 Jul 2023 10:33:20 +0400 Subject: [PATCH 22/43] MDEV-29019 Assertion `(length % 4) == 0' failed in my_lengthsp_utf32 on SELECT Problem: Item_func_conv::val_str() copied the ASCII string with the numeric base conversion result directly to the function result string. In case of a tricky character set (e.g. utf32) it produced an illformed string. Fix: Copy the base conversion result to the function result as is only if the function character set is ASCII compatible, go through a character set conversion otherwise. --- mysql-test/main/ctype_utf32.result | 20 ++++++++++++++++++++ mysql-test/main/ctype_utf32.test | 25 ++++++++++++++++++++++++- sql/item_strfunc.cc | 6 +++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/ctype_utf32.result b/mysql-test/main/ctype_utf32.result index a0f760cbdc0..4ed5ae32a76 100644 --- a/mysql-test/main/ctype_utf32.result +++ b/mysql-test/main/ctype_utf32.result @@ -2940,3 +2940,23 @@ DROP TABLE t1; # # End of 10.2 tests # +# +# Start of 10.4 tests +# +# +# MDEV-29019 Assertion `(length % 4) == 0' failed in my_lengthsp_utf32 on SELECT +# +CREATE TABLE t (a INT); +SET collation_connection=utf32_unicode_ci; +INSERT INTO t VALUES (0); +SELECT * FROM t ORDER BY (OCT(a)); +a +0 +SELECT HEX(OCT(a)) FROM t; +HEX(OCT(a)) +00000030 +DROP TABLE t; +SET NAMES utf8; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/ctype_utf32.test b/mysql-test/main/ctype_utf32.test index a62416621b8..63b6bdd65ae 100644 --- a/mysql-test/main/ctype_utf32.test +++ b/mysql-test/main/ctype_utf32.test @@ -1099,7 +1099,30 @@ CREATE TABLE t1 ( SHOW CREATE TABLE t1; DROP TABLE t1; ---enable_service_connection --echo # --echo # End of 10.2 tests --echo # + + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-29019 Assertion `(length % 4) == 0' failed in my_lengthsp_utf32 on SELECT +--echo # + +CREATE TABLE t (a INT); +SET collation_connection=utf32_unicode_ci; +INSERT INTO t VALUES (0); +SELECT * FROM t ORDER BY (OCT(a)); +SELECT HEX(OCT(a)) FROM t; +DROP TABLE t; +SET NAMES utf8; + + +--echo # +--echo # End of 10.4 tests +--echo # + +--enable_service_connection diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index f95f4795820..ae078dbb22f 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -3533,8 +3533,12 @@ String *Item_func_conv::val_str(String *str) from_base, &endptr, &err); } + uint dummy_errors; if (!(ptr= longlong2str(dec, ans, to_base)) || - str->copy(ans, (uint32) (ptr - ans), default_charset())) + (collation.collation->state & MY_CS_NONASCII) ? + str->copy(ans, (uint32) (ptr - ans), &my_charset_latin1, + collation.collation, &dummy_errors) : + str->copy(ans, (uint32) (ptr - ans), collation.collation)) { null_value= 1; return NULL; From d067de20d6ffce121bcffeefcac19a41b10d1750 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 20 Jul 2023 13:33:14 +1000 Subject: [PATCH 23/43] MDEV-23133 session tracker - warning message typo Concatenation error leads to typo in the warning message --- sql/session_tracker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/session_tracker.cc b/sql/session_tracker.cc index 2f6c5e29bf2..0544827534d 100644 --- a/sql/session_tracker.cc +++ b/sql/session_tracker.cc @@ -167,7 +167,7 @@ bool Session_sysvars_tracker::vars_list::parse_var_list(THD *thd, { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WRONG_VALUE_FOR_VAR, - "%.*s is not a valid system variable and will" + "%.*s is not a valid system variable and will " "be ignored.", (int)var.length, token); } else From a79f4f6ec9dcd213a84eb7f39c8f144153d2725f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 20 Jul 2023 11:06:30 +0400 Subject: [PATCH 24/43] MDEV-22856 Assertion `!str || str != Ptr' and Assertion `!str || str != Ptr || !is_alloced()' failed in String::copy The problem was earlier fixed by MDEV-26953. Adding MTR tests only. --- mysql-test/main/ctype_utf16.result | 56 ++++++++++++++++++++++++++++++ mysql-test/main/ctype_utf16.test | 50 ++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/mysql-test/main/ctype_utf16.result b/mysql-test/main/ctype_utf16.result index 9ec67a67c08..118368da452 100644 --- a/mysql-test/main/ctype_utf16.result +++ b/mysql-test/main/ctype_utf16.result @@ -2841,3 +2841,59 @@ VALUES ('') UNION VALUES ( _utf16 0x0020 COLLATE utf16_bin); # # End of 10.3 tests # +# +# Start of 10.4 tests +# +# +# MDEV-22856 Assertion `!str || str != Ptr' and Assertion `!str || str != Ptr || !is_alloced()' failed in String::copy +# +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +SET sql_buffer_result=1; +CREATE TABLE t(c INT); +INSERT INTO t VALUES(NULL); +SELECT PASSWORD(c) FROM t; +PASSWORD(c) + +DROP TABLE t; +SET sql_buffer_result=DEFAULT; +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +CREATE TABLE t1(c INT); +INSERT INTO t1 VALUES(NULL); +CREATE TABLE t2 AS SELECT PASSWORD(c) FROM t1; +DROP TABLE t2, t1; +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +CREATE TABLE t1 AS SELECT PASSWORD(CAST(NULL AS SIGNED)); +DROP TABLE t1; +SET NAMES utf8mb3, collation_connection='utf16_bin'; +SET @@sql_buffer_result=ON; +CREATE TABLE t (c CHAR(1)); +INSERT INTO t VALUES (1),(1),(1),(NULL); +INSERT INTO t SELECT * FROM t; +SELECT PASSWORD(c) FROM t; +PASSWORD(c) +*E6CC90B878B948C35E92B003C792C46C58C4AF40 +*E6CC90B878B948C35E92B003C792C46C58C4AF40 +*E6CC90B878B948C35E92B003C792C46C58C4AF40 + +*E6CC90B878B948C35E92B003C792C46C58C4AF40 +*E6CC90B878B948C35E92B003C792C46C58C4AF40 +*E6CC90B878B948C35E92B003C792C46C58C4AF40 + +DROP TABLE t; +SET @@sql_buffer_result=DEFAULT; +SET sql_mode=''; +SET SESSION sql_buffer_result=1; +CREATE TABLE t1 (c1 INT); +INSERT INTO t1 VALUES (); +INSERT IGNORE INTO t1 VALUES (NULL); +SET NAMES utf8mb3, collation_connection='utf16_bin'; +SELECT PASSWORD(c1) FROM t1; +PASSWORD(c1) + + +DROP TABLE t1; +SET SESSION sql_buffer_result=DEFAULT; +SET sql_mode=DEFAULT; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/ctype_utf16.test b/mysql-test/main/ctype_utf16.test index cf32fc3094a..a5741d429b3 100644 --- a/mysql-test/main/ctype_utf16.test +++ b/mysql-test/main/ctype_utf16.test @@ -960,3 +960,53 @@ VALUES ('') UNION VALUES ( _utf16 0x0020 COLLATE utf16_bin); --echo # --echo # End of 10.3 tests --echo # + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-22856 Assertion `!str || str != Ptr' and Assertion `!str || str != Ptr || !is_alloced()' failed in String::copy +--echo # + +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +SET sql_buffer_result=1; +CREATE TABLE t(c INT); +INSERT INTO t VALUES(NULL); +SELECT PASSWORD(c) FROM t; +DROP TABLE t; +SET sql_buffer_result=DEFAULT; + +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +CREATE TABLE t1(c INT); +INSERT INTO t1 VALUES(NULL); +CREATE TABLE t2 AS SELECT PASSWORD(c) FROM t1; +DROP TABLE t2, t1; + +SET NAMES utf8mb3, collation_connection='utf16_general_ci'; +CREATE TABLE t1 AS SELECT PASSWORD(CAST(NULL AS SIGNED)); +DROP TABLE t1; + +SET NAMES utf8mb3, collation_connection='utf16_bin'; +SET @@sql_buffer_result=ON; +CREATE TABLE t (c CHAR(1)); +INSERT INTO t VALUES (1),(1),(1),(NULL); +INSERT INTO t SELECT * FROM t; +SELECT PASSWORD(c) FROM t; +DROP TABLE t; +SET @@sql_buffer_result=DEFAULT; + +SET sql_mode=''; +SET SESSION sql_buffer_result=1; +CREATE TABLE t1 (c1 INT); +INSERT INTO t1 VALUES (); +INSERT IGNORE INTO t1 VALUES (NULL); +SET NAMES utf8mb3, collation_connection='utf16_bin'; +SELECT PASSWORD(c1) FROM t1; +DROP TABLE t1; +SET SESSION sql_buffer_result=DEFAULT; +SET sql_mode=DEFAULT; + +--echo # +--echo # End of 10.4 tests +--echo # From 03c2157dd6ad308a825424f61f3829a26873979f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 20 Jul 2023 11:56:19 +0400 Subject: [PATCH 25/43] MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT Also fixes: MDEV-30982 UBSAN: runtime error: null pointer passed as argument 2, which is declared to never be null in my_strnncoll_binary on DELETE Calling memcmp() with a NULL pointer is undefined behaviour according to the C standard, even if the length argument is 0. Adding tests for length==0 before calling memcmp() into: - my_strnncoll_binary() - my_strnncoll_8bit_bin --- mysql-test/main/ctype_binary.result | 34 +++++++++++++++++++++++++++++ mysql-test/main/ctype_binary.test | 33 ++++++++++++++++++++++++++++ mysql-test/main/ctype_latin1.result | 27 +++++++++++++++++++++++ mysql-test/main/ctype_latin1.test | 24 ++++++++++++++++++++ strings/ctype-bin.c | 4 ++-- 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/ctype_binary.result b/mysql-test/main/ctype_binary.result index 02aa08272c1..b43f2136e30 100644 --- a/mysql-test/main/ctype_binary.result +++ b/mysql-test/main/ctype_binary.result @@ -3362,3 +3362,37 @@ DROP FUNCTION f1; # # End of 10.3 tests # +# +# Start of 10.4 tests +# +# +# MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT +# +CREATE TABLE t (c BLOB NOT NULL); +INSERT IGNORE INTO t VALUES (0); +SELECT COUNT(*) FROM t WHERE EXTRACTVALUE(c,'a')='a'; +COUNT(*) +0 +DROP TABLE t; +SET sql_mode=''; +CREATE TABLE t (c TEXT NOT NULL); +INSERT INTO t VALUES(); +Warnings: +Warning 1364 Field 'c' doesn't have a default value +INSERT IGNORE INTO t VALUES (NULL); +Warnings: +Warning 1048 Column 'c' cannot be null +SELECT GROUP_CONCAT(c ORDER BY BINARY c) FROM t GROUP BY c; +GROUP_CONCAT(c ORDER BY BINARY c) +, +DROP TABLE t; +# +# MDEV-30982 UBSAN: runtime error: null pointer passed as argument 2, which is declared to never be null in my_strnncoll_binary on DELETE +# +CREATE TABLE t (c1 SET('1','2','3'),c2 BINARY); +INSERT INTO t VALUES (0,0); +DELETE FROM t WHERE c2 Date: Tue, 27 Jun 2023 12:10:48 +0200 Subject: [PATCH 26/43] MDEV-25237 crash after setting global session_track_system_variables to an invalid value Fix of typo in checking variable list corectness. Fix of error handling in case of variable list parse error --- .../main/mysqltest_tracking_info.result | 30 +++++++++++++++++ mysql-test/main/mysqltest_tracking_info.test | 32 +++++++++++++++++++ .../main/mysqltest_tracking_info_debug.result | 21 ++++++++++++ .../main/mysqltest_tracking_info_debug.test | 30 +++++++++++++++++ sql/session_tracker.cc | 11 +++++-- sql/sys_vars.inl | 4 +++ 6 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 mysql-test/main/mysqltest_tracking_info_debug.result create mode 100644 mysql-test/main/mysqltest_tracking_info_debug.test diff --git a/mysql-test/main/mysqltest_tracking_info.result b/mysql-test/main/mysqltest_tracking_info.result index 3c474cee10f..47e22449912 100644 --- a/mysql-test/main/mysqltest_tracking_info.result +++ b/mysql-test/main/mysqltest_tracking_info.result @@ -57,3 +57,33 @@ ERROR 42000: Variable 'session_track_system_variables' can't be set to the value SET SESSION session_track_system_variables=NULL; ERROR 42000: Variable 'session_track_system_variables' can't be set to the value of 'NULL' # End of 10.3 tests +# +# MDEV-25237: crash after setting global session_track_system_variables +# to an invalid value +# +SET GLOBAL session_track_system_variables='a'; +ERROR HY000: Unknown system variable 'a' +SET GLOBAL event_scheduler=1; +# check that value really returns as it was +set GLOBAL session_track_system_variables='character_set_connection'; +SET GLOBAL session_track_system_variables='a'; +ERROR HY000: Unknown system variable 'a' +connect con,localhost,root,,test; +SET NAMES 'utf8'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_connection +-- utf8 + +SET NAMES 'big5'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_connection +-- big5 + +select @@session_track_system_variables; +@@session_track_system_variables +character_set_connection +connection default; +disconnect con; +SET GLOBAL session_track_system_variables=default; +SET GLOBAL event_scheduler=default; +# End of 10.4 test diff --git a/mysql-test/main/mysqltest_tracking_info.test b/mysql-test/main/mysqltest_tracking_info.test index ae52571b2b9..e73117a8c16 100644 --- a/mysql-test/main/mysqltest_tracking_info.test +++ b/mysql-test/main/mysqltest_tracking_info.test @@ -60,3 +60,35 @@ SET @@GLOBAL.session_track_system_variables=NULL; SET SESSION session_track_system_variables=NULL; --echo # End of 10.3 tests + +--echo # +--echo # MDEV-25237: crash after setting global session_track_system_variables +--echo # to an invalid value +--echo # + +--error ER_UNKNOWN_SYSTEM_VARIABLE +SET GLOBAL session_track_system_variables='a'; +SET GLOBAL event_scheduler=1; + + +--echo # check that value really returns as it was + +set GLOBAL session_track_system_variables='character_set_connection'; +--error ER_UNKNOWN_SYSTEM_VARIABLE +SET GLOBAL session_track_system_variables='a'; + +connect (con,localhost,root,,test); +--enable_session_track_info +SET NAMES 'utf8'; +SET NAMES 'big5'; +--disable_session_track_info + +select @@session_track_system_variables; + +connection default; +disconnect con; + +SET GLOBAL session_track_system_variables=default; +SET GLOBAL event_scheduler=default; + +--echo # End of 10.4 test diff --git a/mysql-test/main/mysqltest_tracking_info_debug.result b/mysql-test/main/mysqltest_tracking_info_debug.result new file mode 100644 index 00000000000..39d17c7ca34 --- /dev/null +++ b/mysql-test/main/mysqltest_tracking_info_debug.result @@ -0,0 +1,21 @@ +set @save_session_track_system_variables=@@session_track_system_variables; +# +# MDEV-25237: Assertion `global_system_variables. +# session_track_system_variables' failed in +# Session_sysvars_tracker::init | SIGSEGV's in __strlen_avx2 | +# UBSAN: runtime error: null pointer passed as argument 1, which +# is declared to never be null in my_strdup +# +# check that that parser problems do not lead to crash +SET @old_debug= @@session.debug; +set debug_dbug="+d,dbug_session_tracker_parse_error"; +SET GLOBAL session_track_system_variables='query_cache_size'; +ERROR HY001: Out of memory; restart server and try again (needed 1 bytes) +set debug_dbug=@old_debug; +SELECT @@global.session_track_system_variables; +@@global.session_track_system_variables +NULL +SET GLOBAL event_scheduler=1; +SET GLOBAL session_track_system_variables=default; +SET GLOBAL event_scheduler=default; +# End of 10.4 test diff --git a/mysql-test/main/mysqltest_tracking_info_debug.test b/mysql-test/main/mysqltest_tracking_info_debug.test new file mode 100644 index 00000000000..1699801f318 --- /dev/null +++ b/mysql-test/main/mysqltest_tracking_info_debug.test @@ -0,0 +1,30 @@ + +--source include/have_debug.inc +--source include/no_protocol.inc +--source include/not_embedded.inc + + +set @save_session_track_system_variables=@@session_track_system_variables; + +--echo # +--echo # MDEV-25237: Assertion `global_system_variables. +--echo # session_track_system_variables' failed in +--echo # Session_sysvars_tracker::init | SIGSEGV's in __strlen_avx2 | +--echo # UBSAN: runtime error: null pointer passed as argument 1, which +--echo # is declared to never be null in my_strdup +--echo # + +--echo # check that that parser problems do not lead to crash +SET @old_debug= @@session.debug; +set debug_dbug="+d,dbug_session_tracker_parse_error"; +--error ER_OUTOFMEMORY +SET GLOBAL session_track_system_variables='query_cache_size'; +set debug_dbug=@old_debug; +SELECT @@global.session_track_system_variables; + +SET GLOBAL event_scheduler=1; + +SET GLOBAL session_track_system_variables=default; +SET GLOBAL event_scheduler=default; + +--echo # End of 10.4 test diff --git a/sql/session_tracker.cc b/sql/session_tracker.cc index 0544827534d..cfc83b8cb74 100644 --- a/sql/session_tracker.cc +++ b/sql/session_tracker.cc @@ -221,7 +221,7 @@ bool sysvartrack_validate_value(THD *thd, const char *str, size_t len) /* Remove leading/trailing whitespace. */ trim_whitespace(system_charset_info, &var); - if (!strcmp(var.str, "*") && !find_sys_var(thd, var.str, var.length)) + if (strcmp(var.str, "*") && !find_sys_var(thd, var.str, var.length)) return true; if (lasts) @@ -331,9 +331,8 @@ void Session_sysvars_tracker::init(THD *thd) mysql_mutex_assert_owner(&LOCK_global_system_variables); DBUG_ASSERT(thd->variables.session_track_system_variables == global_system_variables.session_track_system_variables); - DBUG_ASSERT(global_system_variables.session_track_system_variables); thd->variables.session_track_system_variables= - my_strdup(global_system_variables.session_track_system_variables, + my_strdup(safe_str(global_system_variables.session_track_system_variables), MYF(MY_WME | MY_THREAD_SPECIFIC)); } @@ -572,6 +571,12 @@ bool sysvartrack_global_update(THD *thd, char *str, size_t len) { LEX_STRING tmp= { str, len }; Session_sysvars_tracker::vars_list dummy; + DBUG_EXECUTE_IF("dbug_session_tracker_parse_error", + { + my_error(ER_OUTOFMEMORY, MYF(0), 1); + return true; + }); + if (!dummy.parse_var_list(thd, tmp, false, system_charset_info)) { dummy.construct_var_list(str, len + 1); diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 84d1cd6b331..3e282de439a 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -620,7 +620,11 @@ public: { if (sysvartrack_global_update(thd, new_val, var->save_result.string_value.length)) + { + if (new_val) + my_free(new_val); new_val= 0; + } } global_update_finish(new_val); return (new_val == 0 && var->save_result.string_value.str != 0); From 5a44700aaa1d2fd8867670d9e6a10c085d9856b6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 5 Jul 2023 13:01:43 +1000 Subject: [PATCH 27/43] MDEV-31625 connect engine file_type=DBF insert fails Add the end of file marker x1A to the DBF file and handle it correctly to preserve interoperability with Libreoffice, and others that have followed the DBF spec. The file open mode of "a+" was problematic because Linux and the OSX, the previous main development mode are inconsistent (see man fopen). The main problem per the bug report was the inability to fseek back to the beginning to update the records in the header. As such the "a+" mode is remove and "w+b" is used inserting to a new file and "r+b" is used for appending to the file. In DBFFAM::CloseTableFile move PlugCloseFile down to close the file in all modes. The year unlike the comments is always since 1900. Use the YYYY-MM-DD as an unabigious form during tracing. Thanks for Mr. Zoltan Duna for the descriptive bug report. --- storage/connect/filamdbf.cpp | 53 ++++++++-------- .../connect/mysql-test/connect/r/dbf.result | 60 +++++++++++++++---- storage/connect/mysql-test/connect/t/dbf.test | 10 ++++ 3 files changed, 83 insertions(+), 40 deletions(-) diff --git a/storage/connect/filamdbf.cpp b/storage/connect/filamdbf.cpp index 406974a6924..79eff8b4f68 100644 --- a/storage/connect/filamdbf.cpp +++ b/storage/connect/filamdbf.cpp @@ -335,9 +335,9 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS topt, bool info) hp->Headlen(), hp->Reclen(), fields); htrc("flags(iem)=%d,%d,%d cp=%d\n", hp->Incompleteflag, hp->Encryptflag, hp->Mdxflag, hp->Language); - htrc("%hd records, last changed %02d/%02d/%d\n", - hp->Records(), hp->Filedate[1], hp->Filedate[2], - hp->Filedate[0] + (hp->Filedate[0] <= 30) ? 2000 : 1900); + htrc("%hd records, last changed %04d-%02d-%02d\n", + hp->Records(), + hp->Filedate[0] + 1900, hp->Filedate[1], hp->Filedate[2]); htrc("Field Type Offset Len Dec Set Mdx\n"); } // endif trace @@ -605,8 +605,7 @@ bool DBFFAM::OpenTableFile(PGLOBAL g) strcpy(opmode, (UseTemp) ? "rb" : "r+b"); break; case MODE_INSERT: - // Must be in text mode to remove an eventual EOF character - strcpy(opmode, "a+"); + strcpy(opmode, Records ? "r+b" : "w+b"); break; default: snprintf(g->Message, sizeof(g->Message), MSG(BAD_OPEN_MODE), mode); @@ -619,7 +618,7 @@ bool DBFFAM::OpenTableFile(PGLOBAL g) if (!(Stream = PlugOpenFile(g, filename, opmode))) { if (trace(1)) htrc("%s\n", g->Message); - + return (mode == MODE_READ && errno == ENOENT) ? PushWarning(g, Tdbp) : true; } // endif Stream @@ -643,6 +642,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) { char c; int rc; + int len; MODE mode = Tdbp->GetMode(); Buflen = Blksize; @@ -664,7 +664,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) /************************************************************************/ /* If this is a new file, the header must be generated. */ /************************************************************************/ - int len = GetFileLength(g); + len = GetFileLength(g); if (!len) { // Make the header for this DBF table file @@ -702,7 +702,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) header->Version = DBFTYPE; t = time(NULL) - (time_t)DTVAL::GetShift(); datm = gmtime(&t); - header->Filedate[0] = datm->tm_year - 100; + header->Filedate[0] = datm->tm_year; header->Filedate[1] = datm->tm_mon + 1; header->Filedate[2] = datm->tm_mday; header->SetHeadlen((ushort)hlen); @@ -793,8 +793,12 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) /**************************************************************************/ /* Position the file at the begining of the data. */ /**************************************************************************/ - if (Tdbp->GetMode() == MODE_INSERT) - rc = fseek(Stream, 0, SEEK_END); + if (Tdbp->GetMode() == MODE_INSERT) { + if (len) + rc = fseek(Stream, -1, SEEK_END); + else + rc = fseek(Stream, 0, SEEK_END); + } else rc = fseek(Stream, Headlen, SEEK_SET); @@ -979,6 +983,7 @@ void DBFFAM::CloseTableFile(PGLOBAL g, bool abort) Rbuf = CurNum--; // Closing = true; wrc = WriteBuffer(g); + fputc(0x1a, Stream); } else if (mode == MODE_UPDATE || mode == MODE_DELETE) { if (Modif && !Closing) { // Last updated block remains to be written @@ -1003,35 +1008,27 @@ void DBFFAM::CloseTableFile(PGLOBAL g, bool abort) } // endif's mode if (Tdbp->GetMode() == MODE_INSERT) { - int n = ftell(Stream) - Headlen; - - rc = PlugCloseFile(g, To_Fb); + int n = ftell(Stream) - Headlen - 1; if (n >= 0 && !(n % Lrecl)) { n /= Lrecl; // New number of lines if (n > Records) { // Update the number of rows in the file header - char filename[_MAX_PATH]; + char nRecords[4]; + int4store(nRecords, n); - PlugSetPath(filename, To_File, Tdbp->GetPath()); - if ((Stream= global_fopen(g, MSGID_OPEN_MODE_STRERROR, filename, "r+b"))) - { - char nRecords[4]; - int4store(nRecords, n); - - fseek(Stream, 4, SEEK_SET); // Get header.Records position - fwrite(nRecords, sizeof(nRecords), 1, Stream); - fclose(Stream); - Stream= NULL; - Records= n; // Update Records value - } + fseek(Stream, 4, SEEK_SET); // Get header.Records position + fwrite(nRecords, sizeof(nRecords), 1, Stream); + Stream= NULL; + Records= n; // Update Records value } // endif n } // endif n - } else // Finally close the file - rc = PlugCloseFile(g, To_Fb); + } + // Finally close the file + rc = PlugCloseFile(g, To_Fb); fin: if (trace(1)) diff --git a/storage/connect/mysql-test/connect/r/dbf.result b/storage/connect/mysql-test/connect/r/dbf.result index d8744ef3b70..52ec29bc350 100644 --- a/storage/connect/mysql-test/connect/r/dbf.result +++ b/storage/connect/mysql-test/connect/r/dbf.result @@ -64,6 +64,24 @@ t1 CREATE TABLE `t1` ( `a` int(11) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `TABLE_TYPE`=DBF `FILE_NAME`='t1.dbf' INSERT INTO t1 VALUES (10),(20); +CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); +-------- -------- +FileSize 91 +DBF_Version 03 +NRecords 2 +FirstRecPos 66 +RecLength 12 +TableFlags 0000 +CodePageMark 00 +--- --- +FieldN 0 +Name a +Type N +Offset 0 +Length 11 +Dec 0 +Flags 00 +-------- -------- SELECT * FROM t1; a 10 @@ -89,6 +107,24 @@ t1 CREATE TABLE `t1` ( `a` int(11) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `TABLE_TYPE`=DBF `FILE_NAME`='t1.dbf' `READONLY`=NO INSERT INTO t1 VALUES (30); +CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); +-------- -------- +FileSize 103 +DBF_Version 03 +NRecords 3 +FirstRecPos 66 +RecLength 12 +TableFlags 0000 +CodePageMark 00 +--- --- +FieldN 0 +Name a +Type N +Offset 0 +Length 11 +Dec 0 +Flags 00 +-------- -------- SELECT * FROM t1; a 10 @@ -137,7 +173,7 @@ a test CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 77 +FileSize 78 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -171,7 +207,7 @@ a b c 2 2 2 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 194 +FileSize 195 DBF_Version 03 NRecords 2 FirstRecPos 130 @@ -264,7 +300,7 @@ a -9223372036854775808 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 108 +FileSize 109 DBF_Version 03 NRecords 2 FirstRecPos 66 @@ -308,7 +344,7 @@ a -32768 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 80 +FileSize 81 DBF_Version 03 NRecords 2 FirstRecPos 66 @@ -338,7 +374,7 @@ LENGTH(a) 255 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 322 +FileSize 323 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -419,7 +455,7 @@ a 2001-01-01 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 75 +FileSize 76 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -449,7 +485,7 @@ a 123.0000 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 79 +FileSize 80 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -481,7 +517,7 @@ a 123456789.12345 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 108 +FileSize 109 DBF_Version 03 NRecords 2 FirstRecPos 66 @@ -511,7 +547,7 @@ a 10 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf'); -------- -------- -FileSize 77 +FileSize 78 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -538,7 +574,7 @@ a 10 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf'); -------- -------- -FileSize 77 +FileSize 78 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -567,7 +603,7 @@ a 10 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf'); -------- -------- -FileSize 77 +FileSize 78 DBF_Version 03 NRecords 1 FirstRecPos 66 @@ -604,7 +640,7 @@ c1 c2 i1 i2 30 def 30 123 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf'); -------- -------- -FileSize 291 +FileSize 292 DBF_Version 03 NRecords 3 FirstRecPos 162 diff --git a/storage/connect/mysql-test/connect/t/dbf.test b/storage/connect/mysql-test/connect/t/dbf.test index b798b1a2bc5..50e682c7eb1 100644 --- a/storage/connect/mysql-test/connect/t/dbf.test +++ b/storage/connect/mysql-test/connect/t/dbf.test @@ -63,6 +63,11 @@ DELIMITER ;// CREATE TABLE t1 (a INT NOT NULL) ENGINE=CONNECT TABLE_TYPE=DBF FILE_NAME='t1.dbf'; SHOW CREATE TABLE t1; INSERT INTO t1 VALUES (10),(20); +--chmod 0777 $MYSQLD_DATADIR/test/t1.dbf +--vertical_results +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval CALL dbf_header('$MYSQLD_DATADIR/test/t1.dbf'); +--horizontal_results SELECT * FROM t1; ALTER TABLE t1 READONLY=Yes; SHOW CREATE TABLE t1; @@ -77,6 +82,11 @@ TRUNCATE TABLE t1; ALTER TABLE t1 READONLY=NO; SHOW CREATE TABLE t1; INSERT INTO t1 VALUES (30); +--chmod 0777 $MYSQLD_DATADIR/test/t1.dbf +--vertical_results +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval CALL dbf_header('$MYSQLD_DATADIR/test/t1.dbf'); +--horizontal_results SELECT * FROM t1; DROP TABLE t1; --remove_file $MYSQLD_DATADIR/test/t1.dbf From 620aeb44db8a2f442a662a54a7ea85b363d68dd5 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 14 Jul 2023 14:51:09 +0200 Subject: [PATCH 28/43] MDEV-30159: Client can crash the server with a mysql_list_fields("view") call Do not get value of expensive constants. --- sql/item_cmpfunc.cc | 4 +++- tests/mysql_client_test.c | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index fa96d95adb1..8e6ca318d11 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -6047,7 +6047,9 @@ void Regexp_processor_pcre::fix_owner(Item_func *owner, Item *subject_arg, Item *pattern_arg) { - if (!is_compiled() && pattern_arg->const_item()) + if (!is_compiled() && + pattern_arg->const_item() && + !pattern_arg->is_expensive()) { if (compile(pattern_arg, true)) { diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 21f5c2ecbcb..05190b20624 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -21392,6 +21392,48 @@ static void test_mdev20261() myquery(rc); } +static void test_mdev_30159() +{ + MYSQL_RES *result; + int rc; + + myheader("test_mdev_30159"); + + rc= mysql_query(mysql, "create table t1 (" + " name varchar(100)," + " typ varchar(100)" + ")"); + myquery(rc); + rc= mysql_query(mysql, "insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5)," + "(6,6),(7,7),(8,8),(9,9),(10,10)"); + myquery(rc); + rc= mysql_query(mysql, "insert into t1 values ('', 'value'),('', 'value')"); + myquery(rc); + rc= mysql_query(mysql, "create table t2 (" + " servername varchar(100)" + ")"); + myquery(rc); + rc= mysql_query(mysql, "insert into t2 values (1),(2),(3),(4),(5)," + "(6),(7),(8),(9),(10)"); + myquery(rc); + rc= mysql_query(mysql, "create view v1 as" + " select * from t2" + " where" + " `t2`.`servername` regexp ( select" + " group_concat(`t1`.`name` separator '|')" + " from `t1`" + " where `t1`.`typ`" + " like 'value')"); + myquery(rc); + + result= mysql_list_fields(mysql, "v1", NULL); + mytest(result); + + rc= mysql_query(mysql, "drop view v1"); + myquery(rc); + rc= mysql_query(mysql, "drop table t1, t2"); + myquery(rc); +} static struct my_tests_st my_tests[]= { { "test_mdev_20516", test_mdev_20516 }, @@ -21695,6 +21737,7 @@ static struct my_tests_st my_tests[]= { { "test_mdev_16128", test_mdev_16128 }, { "test_mdev18408", test_mdev18408 }, { "test_mdev20261", test_mdev20261 }, + { "test_mdev_30159", test_mdev_30159 }, { 0, 0 } }; From daeccfcf2b501f2736f3da6421ffa7e053a218e9 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 23 May 2023 10:02:33 +0300 Subject: [PATCH 29/43] Optimized version of safe_strcpy() Note: We should replace most case of safe_strcpy() with strmake() to avoid the not needed zerofill. --- include/m_string.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/m_string.h b/include/m_string.h index 0133d81f6df..7d8ba69c4d1 100644 --- a/include/m_string.h +++ b/include/m_string.h @@ -248,14 +248,15 @@ static inline void lex_string_set3(LEX_CSTRING *lex_str, const char *c_str, */ static inline int safe_strcpy(char *dst, size_t dst_size, const char *src) { - memset(dst, '\0', dst_size); - strncpy(dst, src, dst_size - 1); - /* - If the first condition is true, we are guaranteed to have src length - >= (dst_size - 1), hence safe to access src[dst_size - 1]. - */ - if (dst[dst_size - 2] != '\0' && src[dst_size - 1] != '\0') - return 1; /* Truncation of src. */ + DBUG_ASSERT(dst_size > 0); + /* Note, strncpy will zerofill end of dst if src shorter than dst_size */ + strncpy(dst, src, dst_size); + if (dst[dst_size-1]) + { + /* Ensure string is zero terminated */ + dst[dst_size-1]= 0; + return 1; + } return 0; } From 2ba5c387c1cc0c09e6058adec7f9ef4fd13fe6a0 Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Mon, 10 Jul 2023 13:57:07 -0700 Subject: [PATCH 30/43] Avoid triggering stringop-truncation warning in safe_strcpy The `safe_strcpy()` function was added in https://github.com/mariadb/server/commit/567b68129943#diff-23f88d0b52735bf79b7eb76e2ddbbebc96f3b1ca16e784a347525a9c43134d77 Unfortunately, its current implementation triggers many GCC 8+ string truncation and array bounds warnings, particularly due to the potential for a false positive `-Warray-bounds`. For example, the line `safe_strcpy(delimiter, sizeof(delimiter), ";")` in `client/mysqldump.c` causes the following warning: [1669/1914] Building C object client/CMakeFiles/mariadb-dump.dir/mysqldump.c.o In file included from /PATH/include/my_sys.h:20, from /PATH/mysqldump.c:51: In function ?safe_strcpy?, inlined from ?dump_events_for_db.isra? at /PATH/client/mysqldump.c:2595:3: /PATH/include/m_string.h:258:39: warning: array subscript 1535 is outside array bounds of ?const char[2]? [-Warray-bounds=] 258 | if (dst[dst_size - 2] != '\0' && src[dst_size - 1] != '\0') | ~~~^~~~~~~~~~~~~~ GCC is reporting that the `safe_strcpy` function *could* cause an out-of-bounds read from the constant *source* string `";"`, however this warning is unhelpful and confusing because it can only happen if the size of the *destination* buffer is incorrectly specified, which is not the case here. In https://github.com/MariaDB/server/pull/2640, Andrew Hutchings proposed fixing this by disabling the `-Warray-bounds` check in this function (specifically in https://github.com/MariaDB/server/pull/2640/commits/be382d01d08739d081f6cf40f350f7414f29b49d#diff-23f88d0b52735bf79b7eb76e2ddbbebc96f3b1ca16e784a347525a9c43134d77R255-R262). However, this was rejected because it also disables the *helpful* `-Warray-bounds` check on the destination buffer. Cherry-picking the commit https://github.com/MariaDB/server/commit/a7adfd4c52307876d68ad3386cefd3757ee66e92 from 11.2 by Monty Widenius solves the first two problems: 1. It reimplements `safe_strcpy` a bit more efficiently, skipping the `memset(dst, 0, dst_size)`. This is unnecessary since `strncpy` already pads `dst` with 0 bytes. 2. It will not trigger the `-Warray-bounds` warning, because `src` is not read based on an offset determined from `dst_size`. There is a third problem, however. Using `strncpy` triggers the `-Wstringop-truncation` warning (https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wstringop-truncation), so we need to disable that. However, that is a much less broadly and generally-useful warning so there is no loss of static analysis value caused by disabling it. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- include/m_string.h | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/include/m_string.h b/include/m_string.h index 7d8ba69c4d1..4c8f0074cb9 100644 --- a/include/m_string.h +++ b/include/m_string.h @@ -249,11 +249,30 @@ static inline void lex_string_set3(LEX_CSTRING *lex_str, const char *c_str, static inline int safe_strcpy(char *dst, size_t dst_size, const char *src) { DBUG_ASSERT(dst_size > 0); - /* Note, strncpy will zerofill end of dst if src shorter than dst_size */ + + /* 1) IF there is a 0 byte in the first dst_size bytes of src, strncpy will + * 0-terminate dst, and pad dst with additional 0 bytes out to dst_size. + * + * 2) IF there is no 0 byte in the first dst_size bytes of src, strncpy will + * copy dst_size bytes, and the final byte won't be 0. + * + * In GCC 8+, the `-Wstringop-truncation` warning will object to strncpy() + * being used in this way, so we need to disable this warning for this + * single statement. + */ + +#if defined(__GNUC__) && __GNUC__ >= 8 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif strncpy(dst, src, dst_size); +#if defined(__GNUC__) && __GNUC__ >= 8 +#pragma GCC diagnostic pop +#endif + if (dst[dst_size-1]) { - /* Ensure string is zero terminated */ + /* Only possible in case (2), meaning src was truncated. */ dst[dst_size-1]= 0; return 1; } From add0c01bae4cd2ed435755feb1240a5dbc1c85c1 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:13:59 +0300 Subject: [PATCH 31/43] MDEV-30528 Assertion in dtype_get_at_most_n_mbchars 1. Exclude merging history rows into fts index. The check !history_fts && (index->type & DICT_FTS) was just incorrect attempt to avoid history in fts index. 2. Don't check for duplicates for history rows. --- .../suite/innodb_fts/r/innodb_fts_misc.result | 23 +++++++++++++++++++ .../suite/innodb_fts/t/innodb_fts_misc.test | 18 +++++++++++++++ storage/innobase/row/row0merge.cc | 12 ++++++---- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result index 2cc992be73a..d86d44f38de 100644 --- a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result +++ b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result @@ -1393,3 +1393,26 @@ INSERT INTO t1 VALUES(repeat("this is the test case", 500)); ALTER TABLE t1 KEY_BLOCK_SIZE=4; ALTER TABLE t1 KEY_BLOCK_SIZE=0; DROP TABLE t1; +# +# MDEV-30528 Assertion in dtype_get_at_most_n_mbchars +# +create table t (f text) with system versioning character set utf8 engine=innodb; +insert into t (f) values +('mysql from tutorial dbms stands for database ...') , +('when to use mysql well after that you went through a ...'), +('where will optimizing mysql in what tutorial we will show ...'), +('1001 mysql tricks 1. never run mysqld as root. 2. ...'), +('mysql vs. yoursql in the following database comparison ...'), +('mysql security when configured properly, mysql ...'); +delete from t where f like 'mysql%'; +alter table t add fulltext (f); +select * from t where match(f) against ("use"); +f +when to use mysql well after that you went through a ... +select * from t where match(f) against ("run"); +f +1001 mysql tricks 1. never run mysqld as root. 2. ... +select * from t where match(f) against ("tutorial"); +f +where will optimizing mysql in what tutorial we will show ... +drop table t; diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_misc.test b/mysql-test/suite/innodb_fts/t/innodb_fts_misc.test index 8f4902fd2de..bbc4f089001 100644 --- a/mysql-test/suite/innodb_fts/t/innodb_fts_misc.test +++ b/mysql-test/suite/innodb_fts/t/innodb_fts_misc.test @@ -1341,3 +1341,21 @@ ALTER TABLE t1 KEY_BLOCK_SIZE=4; ALTER TABLE t1 KEY_BLOCK_SIZE=0; DROP TABLE t1; +--echo # +--echo # MDEV-30528 Assertion in dtype_get_at_most_n_mbchars +--echo # +create table t (f text) with system versioning character set utf8 engine=innodb; +insert into t (f) values + ('mysql from tutorial dbms stands for database ...') , + ('when to use mysql well after that you went through a ...'), + ('where will optimizing mysql in what tutorial we will show ...'), + ('1001 mysql tricks 1. never run mysqld as root. 2. ...'), + ('mysql vs. yoursql in the following database comparison ...'), + ('mysql security when configured properly, mysql ...'); +delete from t where f like 'mysql%'; +alter table t add fulltext (f); +select * from t where match(f) against ("use"); +select * from t where match(f) against ("run"); +select * from t where match(f) against ("tutorial"); +# cleanup +drop table t; diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index f0aed489f22..aa431886efe 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -502,7 +502,8 @@ row_merge_buf_add( VCOL_STORAGE vcol_storage; DBUG_ENTER("row_merge_buf_add"); - if (buf->n_tuples >= buf->max_tuples) { + if (buf->n_tuples >= buf->max_tuples + || (history_fts && (buf->index->type & DICT_FTS))) { error: n_row_added = 0; goto end; @@ -595,7 +596,8 @@ error: /* Tokenize and process data for FTS */ - if (!history_fts && (index->type & DICT_FTS)) { + if (index->type & DICT_FTS) { + ut_ad(!history_fts); fts_doc_item_t* doc_item; byte* value; void* ptr; @@ -1876,6 +1878,7 @@ row_merge_read_clustered_index( mach_write_to_8(new_sys_trx_start, trx->id); mach_write_to_8(new_sys_trx_end, TRX_ID_MAX); uint64_t n_rows = 0; + bool history_row = false; /* Scan the clustered index. */ for (;;) { @@ -1892,7 +1895,7 @@ row_merge_read_clustered_index( dtuple_t* row; row_ext_t* ext; page_cur_t* cur = btr_pcur_get_page_cur(&pcur); - bool history_row, history_fts = false; + bool history_fts = false; page_cur_move_to_next(cur); @@ -2527,7 +2530,8 @@ write_buffers: ut_ad(i == 0); break; } - } else if (dict_index_is_unique(buf->index)) { + } else if (!history_row + && dict_index_is_unique(buf->index)) { row_merge_dup_t dup = { buf->index, table, col_map, 0}; From fe618de6919e37fc49c8317b85a1a0b5a42af58b Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:13:59 +0300 Subject: [PATCH 32/43] MDEV-31313 SYSTEM VERSIONING and FOREIGN KEY CASCADE create orphan rows on replica Constraints processing row_ins_check_foreign_constraint() was not called because row_upd_check_references_constraints() didn't see update as delete: node->is_delete was false. Since MDEV-30378 we check for TRG_EVENT_DELETE to detect versioned delete in ha_innobase::update_row(). Now we can use TRG_EVENT_DELETE to set upd_node->is_delete, so constraints processing is triggered correctly. --- mysql-test/suite/versioning/common.inc | 12 +++++ mysql-test/suite/versioning/common_finish.inc | 1 + mysql-test/suite/versioning/r/rpl.result | 49 +++++++++++++++++ mysql-test/suite/versioning/t/rpl.test | 53 +++++++++++++++++++ sql/log_event.cc | 17 +++++- 5 files changed, 130 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/versioning/common.inc b/mysql-test/suite/versioning/common.inc index 25adf15dd50..990fe713d4f 100644 --- a/mysql-test/suite/versioning/common.inc +++ b/mysql-test/suite/versioning/common.inc @@ -91,6 +91,18 @@ begin end~~ delimiter ;~~ +delimiter ~~; +eval create or replace function check_row_slave(row_start $sys_datatype_expl, row_end $sys_datatype_expl) +returns varchar(255) +deterministic +begin + if current_row(row_end) then + return "CURRENT ROW"; + end if; + return "HISTORICAL ROW"; +end~~ +delimiter ;~~ + delimiter ~~; eval create or replace function check_row_ts(row_start timestamp(6), row_end timestamp(6)) returns varchar(255) diff --git a/mysql-test/suite/versioning/common_finish.inc b/mysql-test/suite/versioning/common_finish.inc index 3c4e7b66ff3..f4aa2310a73 100644 --- a/mysql-test/suite/versioning/common_finish.inc +++ b/mysql-test/suite/versioning/common_finish.inc @@ -4,6 +4,7 @@ drop procedure if exists verify_trt; drop procedure if exists verify_trt_dummy; drop function if exists current_row; drop function if exists check_row; +drop function if exists check_row_slave; drop function if exists current_row_ts; drop function if exists check_row_ts; --enable_warnings diff --git a/mysql-test/suite/versioning/r/rpl.result b/mysql-test/suite/versioning/r/rpl.result index 17372c63e99..d4187cd0f3e 100644 --- a/mysql-test/suite/versioning/r/rpl.result +++ b/mysql-test/suite/versioning/r/rpl.result @@ -188,4 +188,53 @@ connection slave; include/diff_tables.inc [master:test.t1,slave:test.t1] connection master; drop table t1; +# +# MDEV-31313 SYSTEM VERSIONING and FOREIGN KEY CASCADE create orphan rows on replica +# +create table parent ( +id int(11) not null auto_increment, +processdate datetime default null, +primary key (id) +) engine=innodb with system versioning; +set timestamp= unix_timestamp('2000-01-01 00:00:00'); +insert into parent values (1, now()); +create table child ( +id int(11) not null auto_increment, +ch_name varchar(30), +andreid int(11) default null, +primary key (id), +key andreid (andreid), +constraint fk_andreid foreign key (andreid) references parent (id) on delete cascade +) engine=innodb with system versioning; +set timestamp= unix_timestamp('2000-01-01 00:00:01'); +insert into child values (null, 'vimtomar', 1); +set timestamp= unix_timestamp('2000-01-01 00:00:02'); +delete from parent where id = 1; +select check_row(row_start, row_end) from parent for system_time all; +check_row(row_start, row_end) +HISTORICAL ROW +select check_row(row_start, row_end) from child for system_time all; +check_row(row_start, row_end) +HISTORICAL ROW +select * from child; +id ch_name andreid +select * from parent; +id processdate +connection slave; +select check_row_slave(row_start, row_end) from parent for system_time all; +check_row_slave(row_start, row_end) +HISTORICAL ROW +select check_row_slave(row_start, row_end) from child for system_time all; +check_row_slave(row_start, row_end) +HISTORICAL ROW +select * from child; +id ch_name andreid +select * from parent; +id processdate +connection master; +set timestamp= default; +drop table child; +drop table parent; +connection slave; +connection master; include/rpl_end.inc diff --git a/mysql-test/suite/versioning/t/rpl.test b/mysql-test/suite/versioning/t/rpl.test index 7d78b60e6fa..2da0931bef0 100644 --- a/mysql-test/suite/versioning/t/rpl.test +++ b/mysql-test/suite/versioning/t/rpl.test @@ -1,4 +1,5 @@ --source suite/versioning/engines.inc +--source suite/versioning/common.inc --source include/have_partition.inc --source include/master-slave.inc @@ -6,6 +7,7 @@ #Testing command counters -BEFORE. #Storing the before counts of Slave connection slave; +--source suite/versioning/common.inc let $slave_com_commit_before= query_get_value(SHOW GLOBAL STATUS LIKE 'com_commit', Value, 1); let $slave_com_insert_before= query_get_value(SHOW GLOBAL STATUS LIKE 'com_insert', Value, 1); let $slave_com_delete_before= query_get_value(SHOW GLOBAL STATUS LIKE 'com_delete', Value, 1); @@ -167,4 +169,55 @@ sync_slave_with_master; connection master; drop table t1; +--echo # +--echo # MDEV-31313 SYSTEM VERSIONING and FOREIGN KEY CASCADE create orphan rows on replica +--echo # +create table parent ( + id int(11) not null auto_increment, + processdate datetime default null, + primary key (id) +) engine=innodb with system versioning; + +set timestamp= unix_timestamp('2000-01-01 00:00:00'); +insert into parent values (1, now()); + +create table child ( + id int(11) not null auto_increment, + ch_name varchar(30), + andreid int(11) default null, + primary key (id), + key andreid (andreid), + constraint fk_andreid foreign key (andreid) references parent (id) on delete cascade +) engine=innodb with system versioning; + +set timestamp= unix_timestamp('2000-01-01 00:00:01'); +insert into child values (null, 'vimtomar', 1); + +set timestamp= unix_timestamp('2000-01-01 00:00:02'); +delete from parent where id = 1; + +select check_row(row_start, row_end) from parent for system_time all; +select check_row(row_start, row_end) from child for system_time all; +select * from child; +select * from parent; + +sync_slave_with_master; + +# Annoying tweaking of microseconds in slave row_end, so row_end can be <= row_start +select check_row_slave(row_start, row_end) from parent for system_time all; +select check_row_slave(row_start, row_end) from child for system_time all; +select * from child; +select * from parent; + +# Cleanup +--source suite/versioning/common_finish.inc +--connection master +set timestamp= default; +drop table child; +drop table parent; + +sync_slave_with_master; +connection master; + +--source suite/versioning/common_finish.inc --source include/rpl_end.inc diff --git a/sql/log_event.cc b/sql/log_event.cc index 181a26ccdb5..23ccaef2a56 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -14759,6 +14759,11 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) return error; } + const bool history_change= m_table->versioned() ? + !m_table->vers_end_field()->is_max() : false; + TABLE_LIST *tl= m_table->pos_in_table_list; + uint8 trg_event_map_save= tl->trg_event_map; + /* This is the situation after locating BI: @@ -14816,9 +14821,17 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) goto err; } - if (m_vers_from_plain && m_table->versioned(VERS_TIMESTAMP)) - m_table->vers_update_fields(); + if (m_table->versioned()) + { + if (m_vers_from_plain && m_table->versioned(VERS_TIMESTAMP)) + m_table->vers_update_fields(); + if (!history_change && !m_table->vers_end_field()->is_max()) + { + tl->trg_event_map|= trg2bit(TRG_EVENT_DELETE); + } + } error= m_table->file->ha_update_row(m_table->record[1], m_table->record[0]); + tl->trg_event_map= trg_event_map_save; if (unlikely(error == HA_ERR_RECORD_IS_THE_SAME)) error= 0; if (m_vers_from_plain && m_table->versioned(VERS_TIMESTAMP)) From c5a83411157e764a599ec69544bf085d435dd84f Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:13:59 +0300 Subject: [PATCH 33/43] MDEV-23100 ODKU of non-versioning column inserts history row Use vers_check_update() to avoid inserting history row for ODKU if now versioned fields specified in update_fields. --- mysql-test/suite/versioning/r/update.result | 14 ++++++++++++++ mysql-test/suite/versioning/t/update.test | 16 ++++++++++++++++ sql/sql_insert.cc | 3 ++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/versioning/r/update.result b/mysql-test/suite/versioning/r/update.result index cbb75a10cf5..f0a5052acf9 100644 --- a/mysql-test/suite/versioning/r/update.result +++ b/mysql-test/suite/versioning/r/update.result @@ -437,4 +437,18 @@ update t1 set a = 3 where b <= 9; update t2 set a = 3 where b <= 9; update t1, t2 set t1.a = 3, t2.a = 3 where t1.b <= 10 and t2.b <= 10 and t1.b = t2.b; drop tables t1, t2; +# +# MDEV-23100 ODKU of non-versioning column inserts history row +# +create table t1 ( +x int unique, +y int without system versioning +) with system versioning; +insert into t1 (x, y) values ('1', '1'); +insert into t1 (x, y) values ('1', '2') +on duplicate key update y = 3; +select x, y, check_row_ts(row_start, row_end) from t1 for system_time all order by row_end; +x y check_row_ts(row_start, row_end) +1 3 CURRENT ROW +drop table t1; # End of 10.4 tests diff --git a/mysql-test/suite/versioning/t/update.test b/mysql-test/suite/versioning/t/update.test index 56bbd909256..b86296616fd 100644 --- a/mysql-test/suite/versioning/t/update.test +++ b/mysql-test/suite/versioning/t/update.test @@ -373,6 +373,22 @@ update t1, t2 set t1.a = 3, t2.a = 3 where t1.b <= 10 and t2.b <= 10 and t1.b = # cleanup drop tables t1, t2; +--echo # +--echo # MDEV-23100 ODKU of non-versioning column inserts history row +--echo # +create table t1 ( + x int unique, + y int without system versioning +) with system versioning; + +insert into t1 (x, y) values ('1', '1'); +insert into t1 (x, y) values ('1', '2') + on duplicate key update y = 3; + +select x, y, check_row_ts(row_start, row_end) from t1 for system_time all order by row_end; + +drop table t1; + --echo # End of 10.4 tests source suite/versioning/common_finish.inc; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 0f1b66f7610..a2c571373cb 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1916,7 +1916,8 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) if (error != HA_ERR_RECORD_IS_THE_SAME) { info->updated++; - if (table->versioned()) + if (table->versioned() && + table->vers_check_update(*info->update_fields)) { if (table->versioned(VERS_TIMESTAMP)) { From 21a8d2c3131379be36b81e1b574ee1c7eaa29599 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:13:59 +0300 Subject: [PATCH 34/43] MDEV-31319 Assertion const_item_cache == true failed in Item_func::fix_fields On create table tmp as select ... we exited Item_func::fix_fields() with error. fix_fields_if_needed('foo' or 'bar') failed and we returned true, but already changed const_item_cache. So the item is in inconsistent state: fixed == false and const_item_cache == false. Now we cleanup the item before the return if Item_func::fix_fields() fails to process. --- mysql-test/suite/vcol/r/vcol_syntax.result | 19 +++++++++++++++++++ mysql-test/suite/vcol/t/vcol_syntax.test | 11 +++++++++++ sql/item_func.cc | 9 +++++++++ 3 files changed, 39 insertions(+) diff --git a/mysql-test/suite/vcol/r/vcol_syntax.result b/mysql-test/suite/vcol/r/vcol_syntax.result index b5f20b19fa4..dd38a1992e5 100644 --- a/mysql-test/suite/vcol/r/vcol_syntax.result +++ b/mysql-test/suite/vcol/r/vcol_syntax.result @@ -197,3 +197,22 @@ Warnings: Warning 1292 Truncated incorrect DECIMAL value: 'x' Warning 1292 Truncated incorrect DECIMAL value: 'test' drop table t1; +# +# MDEV-31319 Assertion const_item_cache == true failed in Item_func::fix_fields +# +create table t (f1 int, f2 int, fv int generated always as (case user() when 'foo' or 'bar' then f1 else f2 end) virtual); +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'foo' +Warning 1292 Truncated incorrect DOUBLE value: 'bar' +select * from t; +f1 f2 fv +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'foo' +Warning 1292 Truncated incorrect DOUBLE value: 'bar' +create table tmp as select * from information_schema.tables where table_name = 't'; +select * from t; +f1 f2 fv +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'foo' +Warning 1292 Truncated incorrect DOUBLE value: 'bar' +drop table t, tmp; diff --git a/mysql-test/suite/vcol/t/vcol_syntax.test b/mysql-test/suite/vcol/t/vcol_syntax.test index 198d61a13aa..3a0ed0270e2 100644 --- a/mysql-test/suite/vcol/t/vcol_syntax.test +++ b/mysql-test/suite/vcol/t/vcol_syntax.test @@ -162,3 +162,14 @@ create table t1 (a int , b date as (1 in ('x' ,(database ()) ))) ; select b from t1; select a from t1 order by 'x' = b; drop table t1; + +--echo # +--echo # MDEV-31319 Assertion const_item_cache == true failed in Item_func::fix_fields +--echo # +create table t (f1 int, f2 int, fv int generated always as (case user() when 'foo' or 'bar' then f1 else f2 end) virtual); +select * from t; +create table tmp as select * from information_schema.tables where table_name = 't'; +select * from t; + +# cleanup +drop table t, tmp; diff --git a/sql/item_func.cc b/sql/item_func.cc index fd393f8de3f..a632ab825e0 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -353,7 +353,10 @@ Item_func::fix_fields(THD *thd, Item **ref) We shouldn't call fix_fields() twice, so check 'fixed' field first */ if ((*arg)->fix_fields_if_needed(thd, arg)) + { + cleanup(); return TRUE; /* purecov: inspected */ + } item= *arg; if (item->maybe_null) @@ -369,9 +372,15 @@ Item_func::fix_fields(THD *thd, Item **ref) } } if (check_arguments()) + { + cleanup(); return true; + } if (fix_length_and_dec()) + { + cleanup(); return TRUE; + } fixed= 1; return FALSE; } From 14cc7e7d6eed0cb3552f8bb28dc8a7a3edf2d89e Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:14:00 +0300 Subject: [PATCH 35/43] MDEV-25644 UPDATE not working properly on transaction precise system versioned table First UPDATE under START TRANSACTION does nothing (nstate= nstate), but anyway generates history. Since update vector is empty we get into (!uvect->n_fields) branch which only adds history row, but does not do update. After that we get current row with wrong (old) row_start value and because of that second UPDATE tries to insert history row again because it sees trx->id != row_start which is the guard to avoid inserting multiple trx_id-based history rows under same transaction (because we have same trx_id and we get duplicate error and this bug demostrates that). But this try anyway fails because PK is based on row_end which is constant under same transaction, so PK didn't change. The fix moves vers_make_update() to an earlier stage of calc_row_difference(). Therefore it prepares update vector before (!uvect->n_fields) check and never gets into that branch, hence no need to handle versioning inside that condition anymore. Now trx->id and row_start are equal after first UPDATE and we don't try to insert second history row. == Cleanups and improvements == ha_innobase::update_row(): vers_set_fields and vers_ins_row are cleaned up into direct condition check. SQLCOM_ALTER_TABLE check now is not used as this is dead code, assertion is done instead. upd_node->is_delete is set in calc_row_difference() just to keep versioning code as much in one place as possible. vers_make_delete() is still located in row_update_for_mysql() as this is required for ha_innodbase::delete_row() as well. row_ins_duplicate_error_in_clust(): Restrict DB_FOREIGN_DUPLICATE_KEY to the better conditions. VERSIONED_DELETE is used specifically to help lower stack to understand what caused current insert. Related to MDEV-29813. --- mysql-test/suite/versioning/r/delete.result | 34 +++++++--- .../suite/versioning/r/update,trx_id.rdiff | 14 ++++- mysql-test/suite/versioning/r/update.result | 33 +++++++++- mysql-test/suite/versioning/t/delete.test | 26 +++++--- mysql-test/suite/versioning/t/update.test | 27 +++++++- sql/sql_update.cc | 5 ++ storage/innobase/handler/ha_innodb.cc | 63 ++++++++++--------- storage/innobase/row/row0ins.cc | 5 +- storage/innobase/row/row0mysql.cc | 8 +-- storage/innobase/row/row0upd.cc | 5 +- 10 files changed, 160 insertions(+), 60 deletions(-) diff --git a/mysql-test/suite/versioning/r/delete.result b/mysql-test/suite/versioning/r/delete.result index 6f8c8921790..89c1ef516eb 100644 --- a/mysql-test/suite/versioning/r/delete.result +++ b/mysql-test/suite/versioning/r/delete.result @@ -133,18 +133,38 @@ drop table t1; # # MDEV-21138 Assertion `col->ord_part' or `f.col->ord_part' failed in row_build_index_entry_low # +# Check DELETE and multi-DELETE with foreign key create table t1 ( f1 int, f2 text, f3 int, fulltext (f2), key(f1), key(f3), -foreign key r (f3) references t1 (f1) on delete set null) +foreign key r (f3) references t1 (f1) on delete set null, +row_start SYS_TYPE as row start invisible, +row_end SYS_TYPE as row end invisible, +period for system_time (row_start, row_end)) with system versioning engine innodb; insert into t1 values (1, repeat('a', 8193), 1), (1, repeat('b', 8193), 1); -select f1, f3, check_row_ts(row_start, row_end) from t1; -f1 f3 check_row_ts(row_start, row_end) +insert into t1 select 2, f2, 2 from t1; +select f1, f3, check_row(row_start, row_end) from t1; +f1 f3 check_row(row_start, row_end) 1 1 CURRENT ROW 1 1 CURRENT ROW -delete from t1; -select f1, f3, check_row_ts(row_start, row_end) from t1 for system_time all; -f1 f3 check_row_ts(row_start, row_end) +2 2 CURRENT ROW +2 2 CURRENT ROW +delete from t1 where f1 = 1; +select f1, f3, check_row(row_start, row_end) from t1 for system_time all order by f1, row_end; +f1 f3 check_row(row_start, row_end) 1 1 HISTORICAL ROW 1 1 HISTORICAL ROW -drop table t1; +2 2 CURRENT ROW +2 2 CURRENT ROW +create table t2 (f1 int); +insert into t2 values (2); +# Multi-delelte +delete t1, t2 from t1 join t2 where t1.f1 = t2.f1; +select f1, f3, check_row(row_start, row_end) from t1 for system_time all order by f1, row_end; +f1 f3 check_row(row_start, row_end) +1 1 HISTORICAL ROW +1 1 HISTORICAL ROW +2 2 HISTORICAL ROW +2 2 HISTORICAL ROW +# Cleanup +drop tables t1, t2; diff --git a/mysql-test/suite/versioning/r/update,trx_id.rdiff b/mysql-test/suite/versioning/r/update,trx_id.rdiff index 7ce75714235..18395507b71 100644 --- a/mysql-test/suite/versioning/r/update,trx_id.rdiff +++ b/mysql-test/suite/versioning/r/update,trx_id.rdiff @@ -1,6 +1,6 @@ ---- update.result 2018-12-19 13:55:35.873917389 +0300 -+++ update,trx_id.reject 2018-12-19 13:55:35.533917399 +0300 -@@ -81,12 +81,10 @@ +--- update.result ++++ update.reject +@@ -84,12 +84,10 @@ commit; select x, y, sys_trx_end = MAXVAL as current from t1 for system_time all order by sys_trx_end, x, y; x y current @@ -14,3 +14,11 @@ 1 1 1 2 2 1 3 3 1 +@@ -464,7 +462,6 @@ + select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; + nid nstate check_row(row_start, row_end) + 1 1 HISTORICAL ROW +-1 1 HISTORICAL ROW + 1 3 CURRENT ROW + commit; + drop tables t1; diff --git a/mysql-test/suite/versioning/r/update.result b/mysql-test/suite/versioning/r/update.result index f0a5052acf9..df463de0f91 100644 --- a/mysql-test/suite/versioning/r/update.result +++ b/mysql-test/suite/versioning/r/update.result @@ -51,19 +51,22 @@ sys_trx_start SYS_DATATYPE as row start invisible, sys_trx_end SYS_DATATYPE as row end invisible, period for system_time (sys_trx_start, sys_trx_end)) with system versioning; +set timestamp= unix_timestamp('2000-01-01 00:00:00'); insert into t1 values(1, 1, 1); -set @ins_t= now(6); select sys_trx_start into @tmp1 from t1; +set timestamp= unix_timestamp('2000-01-01 01:00:00'); update t1 set x= 11, y= 11 where id = 1; select @tmp1 < sys_trx_start as A1, x, y from t1; A1 x y 1 11 11 select sys_trx_start into @tmp1 from t1; +set timestamp= unix_timestamp('2000-01-01 02:00:00'); update t1 set y= 1 where id = 1; select @tmp1 = sys_trx_start as A2, x from t1; A2 x 1 11 drop table t1; +set timestamp= default; create table t1 ( x int, y int, @@ -451,4 +454,32 @@ select x, y, check_row_ts(row_start, row_end) from t1 for system_time all order x y check_row_ts(row_start, row_end) 1 3 CURRENT ROW drop table t1; +# +# MDEV-25644 UPDATE not working properly on transaction precise system versioned table +# +create or replace table t1 (nid int primary key, nstate int, ntype int) engine innodb; +alter table t1 add +row_start SYS_DATATYPE generated always as row start invisible, +add row_end SYS_DATATYPE generated always as row end invisible, +add period for system_time(row_start, row_end), +add system versioning; +insert into t1 values (1, 1, 1); +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +nid nstate check_row(row_start, row_end) +1 1 CURRENT ROW +start transaction; +update t1 set nstate= nstate where nid = 1; +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +nid nstate check_row(row_start, row_end) +1 1 HISTORICAL ROW +1 1 CURRENT ROW +# Bug: ERROR 1761 (23000): Foreign key constraint for table 'xxx', record '1-18446744073709551615' would lead to a duplicate entry in table 'xxx', key 'PRIMARY' +update t1 set nstate= 3 where nid= 1; +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +nid nstate check_row(row_start, row_end) +1 1 HISTORICAL ROW +1 1 HISTORICAL ROW +1 3 CURRENT ROW +commit; +drop tables t1; # End of 10.4 tests diff --git a/mysql-test/suite/versioning/t/delete.test b/mysql-test/suite/versioning/t/delete.test index a5a0497fef2..9debdcfec8c 100644 --- a/mysql-test/suite/versioning/t/delete.test +++ b/mysql-test/suite/versioning/t/delete.test @@ -97,16 +97,26 @@ drop table t1; --echo # --echo # MDEV-21138 Assertion `col->ord_part' or `f.col->ord_part' failed in row_build_index_entry_low --echo # -create table t1 ( +--echo # Check DELETE and multi-DELETE with foreign key +replace_result $sys_datatype_expl SYS_TYPE; +eval create table t1 ( f1 int, f2 text, f3 int, fulltext (f2), key(f1), key(f3), - foreign key r (f3) references t1 (f1) on delete set null) + foreign key r (f3) references t1 (f1) on delete set null, + row_start $sys_datatype_expl as row start invisible, + row_end $sys_datatype_expl as row end invisible, + period for system_time (row_start, row_end)) with system versioning engine innodb; insert into t1 values (1, repeat('a', 8193), 1), (1, repeat('b', 8193), 1); -select f1, f3, check_row_ts(row_start, row_end) from t1; -delete from t1; -select f1, f3, check_row_ts(row_start, row_end) from t1 for system_time all; - -# cleanup -drop table t1; +insert into t1 select 2, f2, 2 from t1; +select f1, f3, check_row(row_start, row_end) from t1; +delete from t1 where f1 = 1; +select f1, f3, check_row(row_start, row_end) from t1 for system_time all order by f1, row_end; +create table t2 (f1 int); +insert into t2 values (2); +--echo # Multi-delelte +delete t1, t2 from t1 join t2 where t1.f1 = t2.f1; +select f1, f3, check_row(row_start, row_end) from t1 for system_time all order by f1, row_end; +--echo # Cleanup +drop tables t1, t2; --source suite/versioning/common_finish.inc diff --git a/mysql-test/suite/versioning/t/update.test b/mysql-test/suite/versioning/t/update.test index b86296616fd..046a4ac149d 100644 --- a/mysql-test/suite/versioning/t/update.test +++ b/mysql-test/suite/versioning/t/update.test @@ -26,15 +26,18 @@ eval create table t1 ( sys_trx_end $sys_datatype_expl as row end invisible, period for system_time (sys_trx_start, sys_trx_end)) with system versioning; +set timestamp= unix_timestamp('2000-01-01 00:00:00'); insert into t1 values(1, 1, 1); -set @ins_t= now(6); select sys_trx_start into @tmp1 from t1; +set timestamp= unix_timestamp('2000-01-01 01:00:00'); update t1 set x= 11, y= 11 where id = 1; select @tmp1 < sys_trx_start as A1, x, y from t1; select sys_trx_start into @tmp1 from t1; +set timestamp= unix_timestamp('2000-01-01 02:00:00'); update t1 set y= 1 where id = 1; select @tmp1 = sys_trx_start as A2, x from t1; drop table t1; +set timestamp= default; replace_result $sys_datatype_expl SYS_DATATYPE; eval create table t1 ( @@ -389,6 +392,28 @@ select x, y, check_row_ts(row_start, row_end) from t1 for system_time all order drop table t1; +--echo # +--echo # MDEV-25644 UPDATE not working properly on transaction precise system versioned table +--echo # +create or replace table t1 (nid int primary key, nstate int, ntype int) engine innodb; +--replace_result $sys_datatype_expl SYS_DATATYPE +eval alter table t1 add + row_start $sys_datatype_expl generated always as row start invisible, + add row_end $sys_datatype_expl generated always as row end invisible, + add period for system_time(row_start, row_end), + add system versioning; +insert into t1 values (1, 1, 1); +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +start transaction; +update t1 set nstate= nstate where nid = 1; +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +--echo # Bug: ERROR 1761 (23000): Foreign key constraint for table 'xxx', record '1-18446744073709551615' would lead to a duplicate entry in table 'xxx', key 'PRIMARY' +update t1 set nstate= 3 where nid= 1; +# Under one transaction trx_id generates only one history row, that differs from timestamp +select nid, nstate, check_row(row_start, row_end) from t1 for system_time all order by row_start, row_end; +commit; +drop tables t1; + --echo # End of 10.4 tests source suite/versioning/common_finish.inc; diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 834fa6111e5..233ad49576e 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -232,6 +232,11 @@ bool TABLE::vers_check_update(List &items) } } } + /* + Tell TRX_ID-versioning that it does not insert history row + (see calc_row_difference()). + */ + vers_write= false; return false; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a787da3fa8..3a8d562f705 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8410,6 +8410,10 @@ calc_row_difference( trx_t* const trx = prebuilt->trx; doc_id_t doc_id = FTS_NULL_DOC_ID; ulint num_v = 0; +#ifndef DBUG_OFF + uint vers_fields = 0; +#endif + prebuilt->versioned_write = table->versioned_write(VERS_TRX_ID); const bool skip_virtual = ha_innobase::omits_virtual_cols(*table->s); ut_ad(!srv_read_only_mode); @@ -8422,6 +8426,14 @@ calc_row_difference( for (uint i = 0; i < table->s->fields; i++) { field = table->field[i]; + +#ifndef DBUG_OFF + if (!field->vers_sys_field() + && !field->vers_update_unversioned()) { + ++vers_fields; + } +#endif + const bool is_virtual = !field->stored_in_db(); if (is_virtual && skip_virtual) { num_v++; @@ -8759,6 +8771,21 @@ calc_row_difference( ut_a(buf <= (byte*) original_upd_buff + buff_len); + const TABLE_LIST *tl= table->pos_in_table_list; + const uint8 op_map= tl->trg_event_map | tl->slave_fk_event_map; + /* Used to avoid reading history in FK check on DELETE (see MDEV-16210). */ + prebuilt->upd_node->is_delete = + (op_map & trg2bit(TRG_EVENT_DELETE) + && table->versioned(VERS_TIMESTAMP)) + ? VERSIONED_DELETE : NO_DELETE; + + if (prebuilt->versioned_write) { + /* Guaranteed by CREATE TABLE, but anyway we make sure we + generate history only when there are versioned fields. */ + DBUG_ASSERT(vers_fields); + prebuilt->upd_node->vers_make_update(trx); + } + ut_ad(uvect->validate()); return(DB_SUCCESS); } @@ -8913,48 +8940,24 @@ ha_innobase::update_row( MySQL that the row is not really updated and it should not increase the count of updated rows. This is fix for http://bugs.mysql.com/29157 */ - if (m_prebuilt->versioned_write - && thd_sql_command(m_user_thd) != SQLCOM_ALTER_TABLE - /* Multiple UPDATE of same rows in single transaction create - historical rows only once. */ - && trx->id != table->vers_start_id()) { - error = row_insert_for_mysql((byte*) old_row, - m_prebuilt, - ROW_INS_HISTORICAL); - if (error != DB_SUCCESS) { - goto func_exit; - } - innobase_srv_conc_exit_innodb(m_prebuilt); - innobase_active_small(); - } DBUG_RETURN(HA_ERR_RECORD_IS_THE_SAME); } else { - const bool vers_set_fields = m_prebuilt->versioned_write - && m_prebuilt->upd_node->update->affects_versioned(); - const bool vers_ins_row = vers_set_fields - && thd_sql_command(m_user_thd) != SQLCOM_ALTER_TABLE; - - TABLE_LIST *tl= table->pos_in_table_list; - uint8 op_map= tl->trg_event_map | tl->slave_fk_event_map; - /* This is not a delete */ - m_prebuilt->upd_node->is_delete = - (vers_set_fields && !vers_ins_row) || - (op_map & trg2bit(TRG_EVENT_DELETE) && - table->versioned(VERS_TIMESTAMP)) - ? VERSIONED_DELETE - : NO_DELETE; - innobase_srv_conc_enter_innodb(m_prebuilt); if (m_prebuilt->upd_node->is_delete) { trx->fts_next_doc_id = 0; } + /* row_start was updated by vers_make_update() + in calc_row_difference() */ error = row_update_for_mysql(m_prebuilt); - if (error == DB_SUCCESS && vers_ins_row + if (error == DB_SUCCESS && m_prebuilt->versioned_write /* Multiple UPDATE of same rows in single transaction create historical rows only once. */ && trx->id != table->vers_start_id()) { + /* UPDATE is not used by ALTER TABLE. Just precaution + as we don't need history generation for ALTER TABLE. */ + ut_ad(thd_sql_command(m_user_thd) != SQLCOM_ALTER_TABLE); error = row_insert_for_mysql((byte*) old_row, m_prebuilt, ROW_INS_HISTORICAL); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 81f662e7a15..6765ba546d5 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2439,7 +2439,10 @@ row_ins_duplicate_error_in_clust( duplicate: trx->error_info = cursor->index; err = DB_DUPLICATE_KEY; - if (cursor->index->table->versioned() + if (thr->prebuilt + && thr->prebuilt->upd_node + && thr->prebuilt->upd_node->is_delete + == VERSIONED_DELETE && entry->vers_history_row()) { ulint trx_id_len; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index bb994a926eb..dc3d0ca2ade 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1790,12 +1790,8 @@ row_update_for_mysql(row_prebuilt_t* prebuilt) ut_ad(!prebuilt->versioned_write || node->table->versioned()); - if (prebuilt->versioned_write) { - if (node->is_delete == VERSIONED_DELETE) { - node->vers_make_delete(trx); - } else if (node->update->affects_versioned()) { - node->vers_make_update(trx); - } + if (prebuilt->versioned_write && node->is_delete == VERSIONED_DELETE) { + node->vers_make_delete(trx); } for (;;) { diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 0ea7aea7fb1..0700792e441 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -3505,9 +3505,8 @@ error_handling: void thd_get_query_start_data(THD *thd, char *buf); /** Appends row_start or row_end field to update vector and sets a -CURRENT_TIMESTAMP/trx->id value to it. -Supposed to be called only by make_versioned_update() and -make_versioned_delete(). +CURRENT_TIMESTAMP/trx->id value to it. Called by vers_make_update() and +vers_make_delete(). @param[in] trx transaction @param[in] vers_sys_idx table->row_start or table->row_end */ void upd_node_t::vers_update_fields(const trx_t *trx, ulint idx) From 3e7561cf35a1241083e88ed530a22aab6712335e Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 20 Jul 2023 14:14:00 +0300 Subject: [PATCH 36/43] MDEV-29357 Assertion (fixed) in Item_func_dayname on INSERT Restrict vcol_cleanup_expr() in close_thread_tables() to only simple locked tables mode. Prelocked is cleaned up like normal statement: in close_thread_table(). --- mysql-test/suite/vcol/r/vcol_syntax.result | 18 ++++++++++++++++++ mysql-test/suite/vcol/t/vcol_syntax.test | 12 ++++++++++++ sql/sql_base.cc | 7 ++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/vcol/r/vcol_syntax.result b/mysql-test/suite/vcol/r/vcol_syntax.result index dd38a1992e5..f1a850e8f7c 100644 --- a/mysql-test/suite/vcol/r/vcol_syntax.result +++ b/mysql-test/suite/vcol/r/vcol_syntax.result @@ -216,3 +216,21 @@ Warnings: Warning 1292 Truncated incorrect DOUBLE value: 'foo' Warning 1292 Truncated incorrect DOUBLE value: 'bar' drop table t, tmp; +# +# MDEV-29357 Assertion (fixed) in Item_func_dayname on INSERT +# +set sql_mode=''; +create table t (c1 blob ,c2 int,c3 char(10) as (dayname (c2))); +create trigger tr before insert on t for each row set new.c2=0; +insert into t values (0, 0, 0); +Warnings: +Warning 1906 The value specified for generated column 'c3' in table 't' has been ignored +Warning 1292 Incorrect datetime value: '0' for column `test`.`t`.`c2` at row 1 +Warning 1292 Incorrect datetime value: '0' for column `test`.`t`.`c2` at row 1 +insert into t values (1, 1, 1); +Warnings: +Warning 1906 The value specified for generated column 'c3' in table 't' has been ignored +Warning 1292 Incorrect datetime value: '1' for column `test`.`t`.`c2` at row 1 +Warning 1292 Incorrect datetime value: '0' for column `test`.`t`.`c2` at row 1 +drop trigger tr; +drop table t; diff --git a/mysql-test/suite/vcol/t/vcol_syntax.test b/mysql-test/suite/vcol/t/vcol_syntax.test index 3a0ed0270e2..cb741bc6def 100644 --- a/mysql-test/suite/vcol/t/vcol_syntax.test +++ b/mysql-test/suite/vcol/t/vcol_syntax.test @@ -173,3 +173,15 @@ select * from t; # cleanup drop table t, tmp; + +--echo # +--echo # MDEV-29357 Assertion (fixed) in Item_func_dayname on INSERT +--echo # +set sql_mode=''; +create table t (c1 blob ,c2 int,c3 char(10) as (dayname (c2))); +create trigger tr before insert on t for each row set new.c2=0; +insert into t values (0, 0, 0); +insert into t values (1, 1, 1); + +drop trigger tr; +drop table t; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 37336a83bfb..47b9c4c6db9 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -913,7 +913,12 @@ void close_thread_tables(THD *thd) !thd->stmt_arena->is_stmt_prepare()) table->part_info->vers_check_limit(thd); #endif - table->vcol_cleanup_expr(thd); + /* + For simple locking we cleanup it here because we don't close thread + tables. For prelocking we close it when we do close thread tables. + */ + if (thd->locked_tables_mode != LTM_PRELOCKED) + table->vcol_cleanup_expr(thd); } /* Detach MERGE children after every statement. Even under LOCK TABLES. */ From 73c9415e6a53d717a5f83ca99f624e0bf1aadb9b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 18 Jul 2023 12:58:58 +1000 Subject: [PATCH 37/43] MDEV-31727: pcre stack size not functioning on clang-16 noinline attribute was being ignored by clang-16 and reporting 32 stack size on Gentoo, 16 locally on Fedora 38. Based on https://stackoverflow.com/questions/54481855/clang-ignoring-attribute-noinline appended noopt in addition to the gcc recognised attributes. After that the -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) returned 1056, simlar to gcc. From https://bugs.gentoo.org/910188. Thanks Zhixu Liu for the great bug report. --- pcre/pcre_exec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcre/pcre_exec.c b/pcre/pcre_exec.c index c30d90acfc7..6df88071c66 100644 --- a/pcre/pcre_exec.c +++ b/pcre/pcre_exec.c @@ -513,7 +513,7 @@ Returns: MATCH_MATCH if matched ) these values are >= 0 static int match(REGISTER PCRE_PUCHAR eptr, REGISTER const pcre_uchar *ecode, PCRE_PUCHAR mstart, int offset_top, match_data *md, eptrblock *eptrb, - unsigned int rdepth) __attribute__((noinline,noclone)); + unsigned int rdepth) __attribute__((optnone,noinline,noclone)); #endif static int match(REGISTER PCRE_PUCHAR eptr, REGISTER const pcre_uchar *ecode, From 8b01c2962b773630b63011f225c253cfe9fcab01 Mon Sep 17 00:00:00 2001 From: Georg Richter Date: Sun, 23 Jul 2023 18:58:26 +0200 Subject: [PATCH 38/43] Remove CLIENT_SSL_VERIFY_SERVER_CERT Since TLS server certificate verification is a client only option, this flag is removed in both client (C/C) and MariaDB server capability flags. This patch reverts commit 89d759b93e3975e5d5e1c5cf9b901c01b9e80ff7 (MySQL Bug #21543) and stores the server certificate validation option in mysql->options.extensions. --- include/mysql_com.h | 8 +++----- include/sql_common.h | 1 + sql-common/client.c | 14 ++++++++------ sql/sql_acl.cc | 1 - 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/mysql_com.h b/include/mysql_com.h index f63cf0ac5c2..2e51f67b662 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -275,7 +275,7 @@ enum enum_indicator_type #define CLIENT_DEPRECATE_EOF (1ULL << 24) #define CLIENT_PROGRESS_OBSOLETE (1ULL << 29) -#define CLIENT_SSL_VERIFY_SERVER_CERT (1ULL << 30) +#define CLIENT_SSL_VERIFY_SERVER_CERT_OBSOLETE (1ULL << 30) /* It used to be that if mysql_real_connect() failed, it would delete any options set by the client, unless the CLIENT_REMEMBER_OPTIONS flag was @@ -326,7 +326,6 @@ enum enum_indicator_type CLIENT_MULTI_STATEMENTS | \ CLIENT_MULTI_RESULTS | \ CLIENT_PS_MULTI_RESULTS | \ - CLIENT_SSL_VERIFY_SERVER_CERT | \ CLIENT_REMEMBER_OPTIONS | \ MARIADB_CLIENT_PROGRESS | \ CLIENT_PLUGIN_AUTH | \ @@ -343,9 +342,8 @@ enum enum_indicator_type If any of the optional flags is supported by the build it will be switched on before sending to the client during the connection handshake. */ -#define CLIENT_BASIC_FLAGS (((CLIENT_ALL_FLAGS & ~CLIENT_SSL) \ - & ~CLIENT_COMPRESS) \ - & ~CLIENT_SSL_VERIFY_SERVER_CERT) +#define CLIENT_BASIC_FLAGS ((CLIENT_ALL_FLAGS & ~CLIENT_SSL) \ + & ~CLIENT_COMPRESS) /** Is raised when a multi-statement transaction diff --git a/include/sql_common.h b/include/sql_common.h index 9836d0c1cdc..a61572e380c 100644 --- a/include/sql_common.h +++ b/include/sql_common.h @@ -44,6 +44,7 @@ struct st_mysql_options_extention { struct mysql_async_context *async_context; HASH connection_attributes; size_t connection_attributes_length; + my_bool tls_verify_server_cert; }; typedef struct st_mysql_methods diff --git a/sql-common/client.c b/sql-common/client.c index a1bdbebf639..6f4ea70a733 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2093,7 +2093,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, If the server does not support ssl, we abort the connection. */ if (mysql->options.use_ssl && - (mysql->client_flag & CLIENT_SSL_VERIFY_SERVER_CERT) && + (mysql->options.extension && mysql->options.extension->tls_verify_server_cert) && !(mysql->server_capabilities & CLIENT_SSL)) { set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate, @@ -2163,7 +2163,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, DBUG_PRINT("info", ("IO layer change done!")); /* Verify server cert */ - if ((mysql->client_flag & CLIENT_SSL_VERIFY_SERVER_CERT) && + if ((mysql->options.extension && mysql->options.extension->tls_verify_server_cert) && ssl_verify_server_cert(net->vio, mysql->host, &cert_error)) { set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate, @@ -3847,10 +3847,12 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg) mysql->options.use_thread_specific_memory= *(my_bool *) arg; break; case MYSQL_OPT_SSL_VERIFY_SERVER_CERT: - if (*(my_bool*) arg) - mysql->options.client_flag|= CLIENT_SSL_VERIFY_SERVER_CERT; - else - mysql->options.client_flag&= ~CLIENT_SSL_VERIFY_SERVER_CERT; + if (!mysql->options.extension) + mysql->options.extension= (struct st_mysql_options_extention *) + my_malloc(sizeof(struct st_mysql_options_extention), + MYF(MY_WME | MY_ZEROFILL)); + if (mysql->options.extension) + mysql->options.extension->tls_verify_server_cert= *(my_bool*) arg; break; case MYSQL_PLUGIN_DIR: EXTENSION_SET_STRING(&mysql->options, plugin_dir, arg); diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index c764d2fe2f7..4c2d063b771 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -12759,7 +12759,6 @@ static bool send_server_handshake_packet(MPVIO_EXT *mpvio, if (ssl_acceptor_fd) { thd->client_capabilities |= CLIENT_SSL; - thd->client_capabilities |= CLIENT_SSL_VERIFY_SERVER_CERT; } if (data_len) From 1c9002cfc85aa175065e3089af771136847335b5 Mon Sep 17 00:00:00 2001 From: Georg Richter Date: Sun, 23 Jul 2023 18:58:26 +0200 Subject: [PATCH 39/43] Remove CLIENT_SSL_VERIFY_SERVER_CERT Since TLS server certificate verification is a client only option, this flag is removed in both client (C/C) and MariaDB server capability flags. This patch reverts commit 89d759b93e3975e5d5e1c5cf9b901c01b9e80ff7 (MySQL Bug #21543) and stores the server certificate validation option in mysql->options.extensions. --- include/mysql_com.h | 8 +++----- include/sql_common.h | 1 + sql-common/client.c | 14 ++++++++------ sql/sql_acl.cc | 1 - 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/mysql_com.h b/include/mysql_com.h index f63cf0ac5c2..2e51f67b662 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -275,7 +275,7 @@ enum enum_indicator_type #define CLIENT_DEPRECATE_EOF (1ULL << 24) #define CLIENT_PROGRESS_OBSOLETE (1ULL << 29) -#define CLIENT_SSL_VERIFY_SERVER_CERT (1ULL << 30) +#define CLIENT_SSL_VERIFY_SERVER_CERT_OBSOLETE (1ULL << 30) /* It used to be that if mysql_real_connect() failed, it would delete any options set by the client, unless the CLIENT_REMEMBER_OPTIONS flag was @@ -326,7 +326,6 @@ enum enum_indicator_type CLIENT_MULTI_STATEMENTS | \ CLIENT_MULTI_RESULTS | \ CLIENT_PS_MULTI_RESULTS | \ - CLIENT_SSL_VERIFY_SERVER_CERT | \ CLIENT_REMEMBER_OPTIONS | \ MARIADB_CLIENT_PROGRESS | \ CLIENT_PLUGIN_AUTH | \ @@ -343,9 +342,8 @@ enum enum_indicator_type If any of the optional flags is supported by the build it will be switched on before sending to the client during the connection handshake. */ -#define CLIENT_BASIC_FLAGS (((CLIENT_ALL_FLAGS & ~CLIENT_SSL) \ - & ~CLIENT_COMPRESS) \ - & ~CLIENT_SSL_VERIFY_SERVER_CERT) +#define CLIENT_BASIC_FLAGS ((CLIENT_ALL_FLAGS & ~CLIENT_SSL) \ + & ~CLIENT_COMPRESS) /** Is raised when a multi-statement transaction diff --git a/include/sql_common.h b/include/sql_common.h index 9836d0c1cdc..a61572e380c 100644 --- a/include/sql_common.h +++ b/include/sql_common.h @@ -44,6 +44,7 @@ struct st_mysql_options_extention { struct mysql_async_context *async_context; HASH connection_attributes; size_t connection_attributes_length; + my_bool tls_verify_server_cert; }; typedef struct st_mysql_methods diff --git a/sql-common/client.c b/sql-common/client.c index a1bdbebf639..6f4ea70a733 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2093,7 +2093,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, If the server does not support ssl, we abort the connection. */ if (mysql->options.use_ssl && - (mysql->client_flag & CLIENT_SSL_VERIFY_SERVER_CERT) && + (mysql->options.extension && mysql->options.extension->tls_verify_server_cert) && !(mysql->server_capabilities & CLIENT_SSL)) { set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate, @@ -2163,7 +2163,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, DBUG_PRINT("info", ("IO layer change done!")); /* Verify server cert */ - if ((mysql->client_flag & CLIENT_SSL_VERIFY_SERVER_CERT) && + if ((mysql->options.extension && mysql->options.extension->tls_verify_server_cert) && ssl_verify_server_cert(net->vio, mysql->host, &cert_error)) { set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate, @@ -3847,10 +3847,12 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg) mysql->options.use_thread_specific_memory= *(my_bool *) arg; break; case MYSQL_OPT_SSL_VERIFY_SERVER_CERT: - if (*(my_bool*) arg) - mysql->options.client_flag|= CLIENT_SSL_VERIFY_SERVER_CERT; - else - mysql->options.client_flag&= ~CLIENT_SSL_VERIFY_SERVER_CERT; + if (!mysql->options.extension) + mysql->options.extension= (struct st_mysql_options_extention *) + my_malloc(sizeof(struct st_mysql_options_extention), + MYF(MY_WME | MY_ZEROFILL)); + if (mysql->options.extension) + mysql->options.extension->tls_verify_server_cert= *(my_bool*) arg; break; case MYSQL_PLUGIN_DIR: EXTENSION_SET_STRING(&mysql->options, plugin_dir, arg); diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index c764d2fe2f7..4c2d063b771 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -12759,7 +12759,6 @@ static bool send_server_handshake_packet(MPVIO_EXT *mpvio, if (ssl_acceptor_fd) { thd->client_capabilities |= CLIENT_SSL; - thd->client_capabilities |= CLIENT_SSL_VERIFY_SERVER_CERT; } if (data_len) From 668eb2ce45c5e9ab062daefcc2119839e197ef72 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 24 Jul 2023 10:38:41 +0200 Subject: [PATCH 40/43] New CC 3.1 --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index d543bed61ba..3393fe35d37 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit d543bed61ba9a117e95764dd1429b21c3e0579d1 +Subproject commit 3393fe35d378744e12636766931cf5109cc6c2e5 From 734583b0d7266f373813e8fcbfc909abf7a84bbd Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 25 Jul 2023 13:10:52 +1000 Subject: [PATCH 41/43] MDEV-31400 Simple plugin dependency resolution We introduce simple plugin dependency. A plugin init function may return HA_ERR_RETRY_INIT. If this happens during server startup when the server is trying to initialise all plugins, the failed plugins will be retried, until no more plugins succeed in initialisation or want to be retried. This will fix spider init bugs which is caused in part by its dependency on Aria for initialisation. The reason we need a new return code, instead of treating every failure as a request for retry, is that it may be impossible to clean up after a failed plugin initialisation. Take InnoDB for example, it has a global variable `buf_page_cleaner_is_active`, which may not satisfy an assertion during a second initialisation try, probably because InnoDB does not expect the initialisation to be called twice. --- include/my_base.h | 1 + sql/handler.cc | 22 +++++--- sql/sql_plugin.cc | 140 ++++++++++++++++++++++++++++++---------------- 3 files changed, 108 insertions(+), 55 deletions(-) diff --git a/include/my_base.h b/include/my_base.h index cc3d57cb0b1..32e3aa06d27 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -442,6 +442,7 @@ enum ha_base_keytype { #define HA_ERR_CRASHED 126 /* Indexfile is crashed */ #define HA_ERR_WRONG_IN_RECORD 127 /* Record-file is crashed */ #define HA_ERR_OUT_OF_MEM 128 /* Out of memory */ +#define HA_ERR_RETRY_INIT 129 /* Initialization failed and should be retried */ #define HA_ERR_NOT_A_TABLE 130 /* not a MYI file - no signature */ #define HA_ERR_WRONG_COMMAND 131 /* Command not supported */ #define HA_ERR_OLD_FILE 132 /* old databasfile */ diff --git a/sql/handler.cc b/sql/handler.cc index 48ce7b3f1f8..2b081df8326 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -550,6 +550,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin) { handlerton *hton; static const char *no_exts[]= { 0 }; + int ret= 0; DBUG_ENTER("ha_initialize_handlerton"); DBUG_PRINT("plugin", ("initialize plugin: '%s'", plugin->name.str)); @@ -559,6 +560,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin) { sql_print_error("Unable to allocate memory for plugin '%s' handlerton.", plugin->name.str); + ret= 1; goto err_no_hton_memory; } @@ -568,12 +570,16 @@ int ha_initialize_handlerton(st_plugin_int *plugin) hton->slot= HA_SLOT_UNDEF; /* Historical Requirement */ plugin->data= hton; // shortcut for the future - if (plugin->plugin->init && plugin->plugin->init(hton)) - { - sql_print_error("Plugin '%s' init function returned error.", - plugin->name.str); + /* [remove after merge] notes on merge conflict (MDEV-31400): + 10.5: 81cd93bbb81d75e7cf14dedede0c9ec0712ace68 + 10.6-10.11: 13ba00ff4933cfc1712676f323587504e453d1b5 + 11.0-11.2: 42f8be10f18163c4025710cf6a212e82bddb2f62 + The 10.11->11.0 conflict is trivial, but the reference commit also + contains different non-conflict changes needs to be applied to 11.0 + (and beyond). + */ + if (plugin->plugin->init && (ret= plugin->plugin->init(hton))) goto err; - } // hton_ext_based_table_discovery() works only when discovery // is supported and the engine if file-based. @@ -616,6 +622,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin) if (idx == (int) DB_TYPE_DEFAULT) { sql_print_warning("Too many storage engines!"); + ret= 1; goto err_deinit; } if (hton->db_type != DB_TYPE_UNKNOWN) @@ -643,6 +650,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin) { sql_print_error("Too many plugins loaded. Limit is %lu. " "Failed on '%s'", (ulong) MAX_HA, plugin->name.str); + ret= 1; goto err_deinit; } hton->slot= total_ha++; @@ -699,7 +707,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin) resolve_sysvar_table_options(hton); update_discovery_counters(hton, 1); - DBUG_RETURN(0); + DBUG_RETURN(ret); err_deinit: /* @@ -717,7 +725,7 @@ err: my_free(hton); err_no_hton_memory: plugin->data= NULL; - DBUG_RETURN(1); + DBUG_RETURN(ret); } int ha_init() diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 1a4d6cba7d3..25ce43fd315 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1411,6 +1411,50 @@ void plugin_unlock_list(THD *thd, plugin_ref *list, uint count) DBUG_VOID_RETURN; } +static void print_init_failed_error(st_plugin_int *p) +{ + sql_print_error("Plugin '%s' registration as a %s failed.", + p->name.str, + plugin_type_names[p->plugin->type].str); +} + +static int plugin_do_initialize(struct st_plugin_int *plugin, uint &state) +{ + DBUG_ENTER("plugin_do_initialize"); + mysql_mutex_assert_not_owner(&LOCK_plugin); + plugin_type_init init= plugin_type_initialize[plugin->plugin->type]; + if (!init) + init= (plugin_type_init) plugin->plugin->init; + if (init) + if (int ret= init(plugin)) + { + /* Plugin init failed and did not requested a retry */ + if (ret != HA_ERR_RETRY_INIT) + print_init_failed_error(plugin); + DBUG_RETURN(ret); + } + state= PLUGIN_IS_READY; // plugin->init() succeeded + + if (plugin->plugin->status_vars) + { + /* + historical ndb behavior caused MySQL plugins to specify + status var names in full, with the plugin name prefix. + this was never fixed in MySQL. + MariaDB fixes that but supports MySQL style too. + */ + SHOW_VAR *show_vars= plugin->plugin->status_vars; + SHOW_VAR tmp_array[2]= {{plugin->plugin->name, + (char *) plugin->plugin->status_vars, SHOW_ARRAY}, + {0, 0, SHOW_UNDEF}}; + if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length)) + show_vars= tmp_array; + + if (add_status_vars(show_vars)) + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin, int *argc, char **argv, bool options_only) @@ -1433,52 +1477,10 @@ static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin, { ret= !options_only && plugin_is_forced(plugin); state= PLUGIN_IS_DISABLED; - goto err; } + else + ret= plugin_do_initialize(plugin, state); - if (plugin_type_initialize[plugin->plugin->type]) - { - if ((*plugin_type_initialize[plugin->plugin->type])(plugin)) - { - sql_print_error("Plugin '%s' registration as a %s failed.", - plugin->name.str, plugin_type_names[plugin->plugin->type].str); - goto err; - } - } - else if (plugin->plugin->init) - { - if (plugin->plugin->init(plugin)) - { - sql_print_error("Plugin '%s' init function returned error.", - plugin->name.str); - goto err; - } - } - state= PLUGIN_IS_READY; // plugin->init() succeeded - - if (plugin->plugin->status_vars) - { - /* - historical ndb behavior caused MySQL plugins to specify - status var names in full, with the plugin name prefix. - this was never fixed in MySQL. - MariaDB fixes that but supports MySQL style too. - */ - SHOW_VAR *show_vars= plugin->plugin->status_vars; - SHOW_VAR tmp_array[2]= { - {plugin->plugin->name, (char*)plugin->plugin->status_vars, SHOW_ARRAY}, - {0, 0, SHOW_UNDEF} - }; - if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length)) - show_vars= tmp_array; - - if (add_status_vars(show_vars)) - goto err; - } - - ret= 0; - -err: if (ret) plugin_variables_deinit(plugin); @@ -1557,7 +1559,7 @@ int plugin_init(int *argc, char **argv, int flags) uint i; struct st_maria_plugin **builtins; struct st_maria_plugin *plugin; - struct st_plugin_int tmp, *plugin_ptr, **reap; + struct st_plugin_int tmp, *plugin_ptr, **reap, **retry_end, **retry_start; MEM_ROOT tmp_root; bool reaped_mandatory_plugin= false; bool mandatory= true; @@ -1707,11 +1709,16 @@ int plugin_init(int *argc, char **argv, int flags) */ mysql_mutex_lock(&LOCK_plugin); + /* List of plugins to reap */ reap= (st_plugin_int **) my_alloca((plugin_array.elements+1) * sizeof(void*)); *(reap++)= NULL; + /* List of plugins to retry */ + retry_start= retry_end= + (st_plugin_int **) my_alloca((plugin_array.elements+1) * sizeof(void*)); for(;;) { + int error; for (i=0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++) { HASH *hash= plugin_hash + plugin_type_initialization_order[i]; @@ -1720,15 +1727,51 @@ int plugin_init(int *argc, char **argv, int flags) plugin_ptr= (struct st_plugin_int *) my_hash_element(hash, idx); if (plugin_ptr->state == PLUGIN_IS_UNINITIALIZED) { - if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, - (flags & PLUGIN_INIT_SKIP_INITIALIZATION))) + error= plugin_initialize(&tmp_root, plugin_ptr, argc, argv, + (flags & PLUGIN_INIT_SKIP_INITIALIZATION)); + if (error) { plugin_ptr->state= PLUGIN_IS_DYING; - *(reap++)= plugin_ptr; + /* The plugin wants a retry of the initialisation, + possibly due to dependency on other plugins */ + if (unlikely(error == HA_ERR_RETRY_INIT)) + *(retry_end++)= plugin_ptr; + else + *(reap++)= plugin_ptr; } } } } + /* Retry plugins that asked for it */ + while (retry_start < retry_end) + { + st_plugin_int **to_re_retry, **retrying; + for (to_re_retry= retrying= retry_start; retrying < retry_end; retrying++) + { + plugin_ptr= *retrying; + uint state= plugin_ptr->state; + mysql_mutex_unlock(&LOCK_plugin); + error= plugin_do_initialize(plugin_ptr, state); + mysql_mutex_lock(&LOCK_plugin); + plugin_ptr->state= state; + if (error == HA_ERR_RETRY_INIT) + *(to_re_retry++)= plugin_ptr; + else if (error) + *(reap++)= plugin_ptr; + } + /* If the retry list has not changed, i.e. if all retry attempts + result in another retry request, empty the retry list */ + if (to_re_retry == retry_end) + while (to_re_retry > retry_start) + { + plugin_ptr= *(--to_re_retry); + *(reap++)= plugin_ptr; + /** `plugin_do_initialize()' did not print any error in this + case, so we do it here. */ + print_init_failed_error(plugin_ptr); + } + retry_end= to_re_retry; + } /* load and init plugins from the plugin table (unless done already) */ if (flags & PLUGIN_INIT_SKIP_PLUGIN_TABLE) @@ -1754,6 +1797,7 @@ int plugin_init(int *argc, char **argv, int flags) } mysql_mutex_unlock(&LOCK_plugin); + my_afree(retry_start); my_afree(reap); if (reaped_mandatory_plugin) goto err; From 063f4ac25e7d1d689b2d6447fbaab2aaf3ba0750 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Wed, 28 Jun 2023 10:28:31 -0600 Subject: [PATCH 42/43] MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers MDEV-31749 sporadic assert in MDEV-30619 new test If the workers of a parallel replica are busy (potentially with long queues), but the SQL thread has no events left to distribute (so it goes idle), then the next event that comes from the primary will update mi->last_master_timestamp with its timestamp, even if the workers have not yet finished. This patch changes the parallel replica logic which updates last_master_timestamp after idling from using solely sql_thread_caught_up (added in MDEV-29639) to using the latter with rli queued/dequeued event counters. That is, if the queued count is equal to the dequeued count, it means all events have been processed and the replica is considered idle when the driver thread has also distributed all events. Low level details of the commit include - to make a more generalized test for Seconds_Behind_Master on the parallel replica, rpl_delayed_parallel_slave_sbm.test is renamed to rpl_parallel_sbm.test for this purpose. - pause_sql_thread_on_next_event usage was removed with the MDEV-30619 fixes. Rather than remove it, we adapt it to the needs of this test case - added test case to cover SBM spike of relay log read and LMT update that was fixed by MDEV-29639 - rpl_seconds_behind_master_spike.test is made to use the negate_clock_diff_with_master debug eval. Reviewed By: ============ Andrei Elkin --- ...ave_sbm.result => rpl_parallel_sbm.result} | 63 +++++--- .../r/rpl_seconds_behind_master_spike.result | 29 +++- .../rpl_delayed_parallel_slave_sbm-slave.opt | 1 - .../suite/rpl/t/rpl_parallel_sbm-slave.opt | 1 + ...l_slave_sbm.test => rpl_parallel_sbm.test} | 135 ++++++++++++------ .../t/rpl_seconds_behind_master_spike.test | 90 +++++++++++- sql/slave.cc | 37 ++++- 7 files changed, 275 insertions(+), 81 deletions(-) rename mysql-test/suite/rpl/r/{rpl_delayed_parallel_slave_sbm.result => rpl_parallel_sbm.result} (54%) delete mode 100644 mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_sbm-slave.opt rename mysql-test/suite/rpl/t/{rpl_delayed_parallel_slave_sbm.test => rpl_parallel_sbm.test} (56%) diff --git a/mysql-test/suite/rpl/r/rpl_delayed_parallel_slave_sbm.result b/mysql-test/suite/rpl/r/rpl_parallel_sbm.result similarity index 54% rename from mysql-test/suite/rpl/r/rpl_delayed_parallel_slave_sbm.result rename to mysql-test/suite/rpl/r/rpl_parallel_sbm.result index b00a8a5e1d7..f3cc8454510 100644 --- a/mysql-test/suite/rpl/r/rpl_delayed_parallel_slave_sbm.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_sbm.result @@ -1,12 +1,17 @@ include/master-slave.inc [connection master] +# +# MDEV-29639: Seconds_Behind_Master is incorrect for Delayed, Parallel Replicas +# connection slave; include/stop_slave.inc +set @@GLOBAL.debug_dbug= "d,negate_clock_diff_with_master"; +set @@GLOBAL.slave_parallel_mode= CONSERVATIVE; change master to master_delay=3, master_use_gtid=Slave_Pos; -set @@GLOBAL.slave_parallel_threads=2; include/start_slave.inc connection master; create table t1 (a int); +create table t2 (a int); include/sync_slave_sql_with_master.inc # # Pt 1) Ensure SBM is updated immediately upon arrival of the next event @@ -25,11 +30,10 @@ connection slave; UNLOCK TABLES; include/sync_with_master_gtid.inc # -# Pt 2) If the SQL thread has not entered an idle state, ensure +# Pt 2) If the worker threads have not entered an idle state, ensure # following events do not update SBM -# Stop slave IO thread so it receives both events together on restart connection slave; -include/stop_slave_io.inc +LOCK TABLES t1 WRITE; connection master; # Sleep 2 to allow a buffer between events for SBM check insert into t1 values (1); @@ -37,36 +41,49 @@ insert into t1 values (1); insert into t1 values (2); include/save_master_pos.inc connection slave; -LOCK TABLES t1 WRITE; -SET @@global.debug_dbug="+d,pause_sql_thread_on_next_event"; -START SLAVE IO_THREAD; -# Before we start processing the events, we ensure both transactions -# were written into the relay log. Otherwise, if the IO thread takes too -# long to queue the events, the sql thread can think it has caught up -# too quickly. -SET DEBUG_SYNC='now WAIT_FOR paused_on_event'; -include/sync_io_with_master.inc -SET @@global.debug_dbug="-d,pause_sql_thread_on_next_event"; -SET DEBUG_SYNC='now SIGNAL sql_thread_continue'; # Wait for first transaction to complete SQL delay and begin execution.. -# Validate SBM calculation doesn't use the second transaction because SQL thread shouldn't have gone idle.. +# Validate SBM calculation doesn't use the second transaction because worker threads shouldn't have gone idle.. # ..and that SBM wasn't calculated using prior committed transactions # ..done connection slave; UNLOCK TABLES; -# +include/wait_for_slave_param.inc [Relay_Master_Log_File] +include/wait_for_slave_param.inc [Exec_Master_Log_Pos] # Cleanup -# Reset master_delay include/stop_slave.inc CHANGE MASTER TO master_delay=0; -set @@GLOBAL.slave_parallel_threads=4; -SET @@global.debug_dbug=""; -SET DEBUG_SYNC='RESET'; include/start_slave.inc +# +# MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers +# +connection slave; +# Ensure the replica is fully idle before starting transactions +# Lock t1 on slave so the first received transaction does not complete/commit +LOCK TABLES t1 WRITE; connection master; -DROP TABLE t1; +insert into t1 values (3); +include/save_master_gtid.inc +connection slave; +# Waiting for first transaction to begin.. +connection master; +# Sleep 2 sec to create a gap between events +INSERT INTO t2 VALUES (1); +include/save_master_gtid.inc +connection slave; +# Waiting for second transaction to begin.. +connection slave; +UNLOCK TABLES; +include/sync_with_master_gtid.inc +# +# Cleanup +connection master; +DROP TABLE t1, t2; include/save_master_gtid.inc connection slave; include/sync_with_master_gtid.inc +include/stop_slave.inc +set @@GLOBAL.debug_dbug= ""; +set @@GLOBAL.slave_parallel_mode= "$save_parallel_mode"; +include/start_slave.inc include/rpl_end.inc -# End of rpl_delayed_parallel_slave_sbm.test +# End of rpl_parallel_sbm.test diff --git a/mysql-test/suite/rpl/r/rpl_seconds_behind_master_spike.result b/mysql-test/suite/rpl/r/rpl_seconds_behind_master_spike.result index 4eeb863bb40..bb71f6c92b0 100644 --- a/mysql-test/suite/rpl/r/rpl_seconds_behind_master_spike.result +++ b/mysql-test/suite/rpl/r/rpl_seconds_behind_master_spike.result @@ -3,7 +3,7 @@ include/master-slave.inc connection slave; include/stop_slave.inc SET @save_dbug= @@GLOBAL.debug_dbug; -SET @@global.debug_dbug="+d,pause_sql_thread_on_fde"; +SET @@global.debug_dbug="+d,pause_sql_thread_on_fde,negate_clock_diff_with_master"; include/start_slave.inc # Future events must be logged at least 2 seconds after # the slave starts @@ -34,8 +34,31 @@ SET @@global.debug_dbug="-d,pause_sql_thread_on_fde"; SET DEBUG_SYNC='now SIGNAL sql_thread_continue'; # Wait for SQL thread to continue into normal execution SET DEBUG_SYNC='RESET'; +# +# MDEV-29639 +# When receiving an event after the SQL Thread idles, +# Seconds_Behind_Master should not update before it updates +# last_master_timestamp +connection slave; +include/stop_slave.inc +set @@global.debug_dbug="+d,pause_sql_thread_on_next_event"; +include/start_slave.inc +connection master; +insert into t1 values(2); +include/save_master_gtid.inc +connection slave; +set debug_sync='now wait_for paused_on_event'; +connection master; +# Sleeping 1s to create a visible SBM gap between events +insert into t1 values(3); +include/save_master_gtid.inc +connection slave; +set debug_sync='now wait_for paused_on_event'; +include/stop_slave.inc +set debug_sync='RESET'; +SET @@global.debug_dbug=$save_dbug; +include/start_slave.inc +include/sync_with_master_gtid.inc connection master; DROP TABLE t1; -connection slave; -SET @@global.debug_dbug=$save_dbug; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm-slave.opt b/mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm-slave.opt deleted file mode 100644 index 9eea6a54b68..00000000000 --- a/mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm-slave.opt +++ /dev/null @@ -1 +0,0 @@ ---slave-parallel-threads=4 diff --git a/mysql-test/suite/rpl/t/rpl_parallel_sbm-slave.opt b/mysql-test/suite/rpl/t/rpl_parallel_sbm-slave.opt new file mode 100644 index 00000000000..c3407cde66b --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_sbm-slave.opt @@ -0,0 +1 @@ +--slave-parallel-threads=2 diff --git a/mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm.test b/mysql-test/suite/rpl/t/rpl_parallel_sbm.test similarity index 56% rename from mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm.test rename to mysql-test/suite/rpl/t/rpl_parallel_sbm.test index 1ac1bc14468..e738f55e7ec 100644 --- a/mysql-test/suite/rpl/t/rpl_delayed_parallel_slave_sbm.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_sbm.test @@ -1,4 +1,14 @@ # +# Ensure that Seconds_Behind_Master works correctly on the parallel replica. +# +--source include/master-slave.inc +--source include/have_log_bin.inc +--source include/have_debug.inc + +--echo # +--echo # MDEV-29639: Seconds_Behind_Master is incorrect for Delayed, Parallel Replicas +--echo # + # This test ensures that after a delayed parallel slave has idled, i.e. # executed everything in its relay log, the next event group that the SQL # thread reads from the relay log will immediately be used in the @@ -6,26 +16,21 @@ # Seconds_Behind_Master is based on the timestamp of the new transaction, # rather than the last committed transaction. # -# References: -# MDEV-29639: Seconds_Behind_Master is incorrect for Delayed, Parallel -# Replicas -# - ---source include/master-slave.inc ---source include/have_debug.inc ---source include/have_debug_sync.inc --connection slave --source include/stop_slave.inc ---let $old_debug_dbug= `SELECT @@global.debug_dbug` +--let $save_dbug= `SELECT @@GLOBAL.debug_dbug` +--let $save_parallel_mode= `SELECT @@GLOBAL.slave_parallel_mode` +set @@GLOBAL.debug_dbug= "d,negate_clock_diff_with_master"; +set @@GLOBAL.slave_parallel_mode= CONSERVATIVE; --let $master_delay= 3 --eval change master to master_delay=$master_delay, master_use_gtid=Slave_Pos ---let $old_slave_threads= `SELECT @@GLOBAL.slave_parallel_threads` -set @@GLOBAL.slave_parallel_threads=2; --source include/start_slave.inc --connection master +--let insert_ctr= 0 create table t1 (a int); +create table t2 (a int); --source include/sync_slave_sql_with_master.inc --echo # @@ -40,7 +45,6 @@ LOCK TABLES t1 WRITE; sleep 2; --let $ts_trx_before_ins= `SELECT UNIX_TIMESTAMP()` ---let insert_ctr= 0 --eval insert into t1 values ($insert_ctr) --inc $insert_ctr --source include/save_master_gtid.inc @@ -66,15 +70,13 @@ UNLOCK TABLES; --source include/sync_with_master_gtid.inc --echo # ---echo # Pt 2) If the SQL thread has not entered an idle state, ensure +--echo # Pt 2) If the worker threads have not entered an idle state, ensure --echo # following events do not update SBM ---echo # Stop slave IO thread so it receives both events together on restart --connection slave ---source include/stop_slave_io.inc +LOCK TABLES t1 WRITE; --connection master - --echo # Sleep 2 to allow a buffer between events for SBM check sleep 2; --let $ts_trxpt2_before_ins= `SELECT UNIX_TIMESTAMP()` @@ -88,29 +90,14 @@ sleep 3; --source include/save_master_pos.inc --connection slave -LOCK TABLES t1 WRITE; - -SET @@global.debug_dbug="+d,pause_sql_thread_on_next_event"; - -START SLAVE IO_THREAD; - ---echo # Before we start processing the events, we ensure both transactions ---echo # were written into the relay log. Otherwise, if the IO thread takes too ---echo # long to queue the events, the sql thread can think it has caught up ---echo # too quickly. -SET DEBUG_SYNC='now WAIT_FOR paused_on_event'; ---source include/sync_io_with_master.inc -SET @@global.debug_dbug="-d,pause_sql_thread_on_next_event"; -SET DEBUG_SYNC='now SIGNAL sql_thread_continue'; - --echo # Wait for first transaction to complete SQL delay and begin execution.. --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock%' AND command LIKE 'Slave_Worker'; --source include/wait_condition.inc ---echo # Validate SBM calculation doesn't use the second transaction because SQL thread shouldn't have gone idle.. +--echo # Validate SBM calculation doesn't use the second transaction because worker threads shouldn't have gone idle.. --let $sbm_after_trx_no_idle= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1) --let $timestamp_trxpt2_arrive= `SELECT UNIX_TIMESTAMP()` -if (`SELECT $sbm_after_trx_no_idle < $timestamp_trxpt2_arrive - $ts_trx_after_ins`) +if (`SELECT $sbm_after_trx_no_idle < $timestamp_trxpt2_arrive - $ts_trx_after_ins - 1`) { --let $cmpv= `SELECT $timestamp_trxpt2_arrive - $ts_trx_after_ins` --echo # SBM $sbm_after_trx_no_idle was more recent than time since last transaction ($cmpv seconds) @@ -127,24 +114,86 @@ if (`SELECT $sbm_after_trx_no_idle > ($seconds_since_idling + 1)`) --connection slave UNLOCK TABLES; +--source include/sync_with_master.inc + +--echo # Cleanup +--source include/stop_slave.inc +--eval CHANGE MASTER TO master_delay=0 +--source include/start_slave.inc + + +--echo # +--echo # MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers +--echo # + +# This test ensures that a parallel slave will not update +# Seconds_Behind_Master after the SQL Thread has idled if the worker threads +# are still executing events. To test this, two events are executed on the +# primary with $sleep seconds in-between them. Once the second event begins +# execution on the replica, Seconds_Behind_Master is queried to ensure it +# reflects the value of the first transaction, rather than the second. + +--connection slave +--echo # Ensure the replica is fully idle before starting transactions +--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Slave has read all relay log%'; +--source include/wait_condition.inc +--let $wait_condition= SELECT count(*)=2 FROM information_schema.processlist WHERE state LIKE 'Waiting for work from SQL thread'; +--source include/wait_condition.inc + +--echo # Lock t1 on slave so the first received transaction does not complete/commit +LOCK TABLES t1 WRITE; + +--connection master +--let $ts_t1_before_master_ins= `SELECT UNIX_TIMESTAMP()` +--eval insert into t1 values ($insert_ctr) +--inc $insert_ctr +--source include/save_master_gtid.inc + +--connection slave +--echo # Waiting for first transaction to begin.. +--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock'; +--source include/wait_condition.inc + +--let $sbm_1= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1) + +--connection master +--let $sleep = 2 +--echo # Sleep $sleep sec to create a gap between events +sleep $sleep; +INSERT INTO t2 VALUES (1); +--source include/save_master_gtid.inc + +--connection slave +--echo # Waiting for second transaction to begin.. +--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%'; +--source include/wait_condition.inc + +--let $sbm_2= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1) + +if (`SELECT $sbm_1 + $sleep > $sbm_2`) +{ + --echo # Seconds_Behind_Masters: $sbm_1 $sbm_2_0 + --die Two successive Seconds_Behind_Master timestamps must be separated by the sleep parameter value or greater +} + +--connection slave +UNLOCK TABLES; +--source include/sync_with_master_gtid.inc + --echo # --echo # Cleanup ---echo # Reset master_delay ---source include/stop_slave.inc ---eval CHANGE MASTER TO master_delay=0 ---eval set @@GLOBAL.slave_parallel_threads=$old_slave_threads ---eval SET @@global.debug_dbug="$old_debug_dbug" -SET DEBUG_SYNC='RESET'; ---source include/start_slave.inc - --connection master -DROP TABLE t1; +DROP TABLE t1, t2; --source include/save_master_gtid.inc --connection slave --source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +--eval set @@GLOBAL.debug_dbug= "$save_dbug" +--evalp set @@GLOBAL.slave_parallel_mode= "$save_parallel_mode" +--source include/start_slave.inc --source include/rpl_end.inc ---echo # End of rpl_delayed_parallel_slave_sbm.test +--echo # End of rpl_parallel_sbm.test diff --git a/mysql-test/suite/rpl/t/rpl_seconds_behind_master_spike.test b/mysql-test/suite/rpl/t/rpl_seconds_behind_master_spike.test index d1bfbf25bae..d96863a14c2 100644 --- a/mysql-test/suite/rpl/t/rpl_seconds_behind_master_spike.test +++ b/mysql-test/suite/rpl/t/rpl_seconds_behind_master_spike.test @@ -27,7 +27,7 @@ --connection slave --source include/stop_slave.inc SET @save_dbug= @@GLOBAL.debug_dbug; -SET @@global.debug_dbug="+d,pause_sql_thread_on_fde"; +SET @@global.debug_dbug="+d,pause_sql_thread_on_fde,negate_clock_diff_with_master"; --source include/start_slave.inc --let $sleep_time=2 @@ -93,11 +93,93 @@ SET DEBUG_SYNC='now SIGNAL sql_thread_continue'; # Reset last sql_thread_continue signal SET DEBUG_SYNC='RESET'; + +--echo # +--echo # MDEV-29639 +--echo # When receiving an event after the SQL Thread idles, +--echo # Seconds_Behind_Master should not update before it updates +--echo # last_master_timestamp + +--connection slave +--source include/stop_slave.inc +set @@global.debug_dbug="+d,pause_sql_thread_on_next_event"; +--source include/start_slave.inc + +--connection master +insert into t1 values(2); +--source include/save_master_gtid.inc + +# Each event after starting will trigger a pause, so continually send signal +# sql_thread_continue until caught up +--connection slave +--let $caught_up=0 +--let $tries= 0 +set debug_sync='now wait_for paused_on_event'; +--disable_query_log +while (!$caught_up) +{ + set debug_sync='now signal sql_thread_continue'; + --let $slave_gtid= `SELECT @@global.gtid_slave_pos` + if (`SELECT strcmp("$master_pos","$slave_gtid") = 0`) + { + --inc $caught_up + } + --inc $tries + # Wait 30s + if (`SELECT $tries > 300`) + { + --die Replica failed to sync with primary + } + sleep 0.1; +} +--enable_query_log + +--connection master +--echo # Sleeping 1s to create a visible SBM gap between events +sleep 1; +insert into t1 values(3); +--source include/save_master_gtid.inc + +--connection slave +set debug_sync='now wait_for paused_on_event'; +--let $sbm= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1) + +if ($sbm) +{ + --echo # Expected Seconds_Behind_Master to be 0 but was $sbm + --die Seconds_Behind_Master should not show updates before last_master_timestamp is updated +} + +# Continually send signal sql_thread_continue until caught up +--let $caught_up=0 +--let $tries= 0 +--disable_query_log +while (!$caught_up) +{ + set debug_sync='now signal sql_thread_continue'; + --let $slave_gtid= `SELECT @@global.gtid_slave_pos` + if (`SELECT strcmp("$master_pos","$slave_gtid") = 0`) + { + --inc $caught_up + } + --inc $tries + # Wait 30s + if (`SELECT $tries > 300`) + { + --die Replica failed to sync with primary + } + sleep 0.1; +} +--enable_query_log + # Cleanup +--source include/stop_slave.inc +set debug_sync='RESET'; +SET @@global.debug_dbug=$save_dbug; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + --connection master DROP TABLE t1; ---connection slave -SET @@global.debug_dbug=$save_dbug; - --source include/rpl_end.inc diff --git a/sql/slave.cc b/sql/slave.cc index f43240a8866..0b575b5f714 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -1857,8 +1857,10 @@ static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi) (master_row= mysql_fetch_row(master_res))) { mysql_mutex_lock(&mi->data_lock); - mi->clock_diff_with_master= - (long) (time((time_t*) 0) - strtoul(master_row[0], 0, 10)); + mi->clock_diff_with_master= DBUG_EVALUATE_IF( + "negate_clock_diff_with_master", 0, + (long) (time((time_t *) 0) - strtoul(master_row[0], 0, 10))); + mysql_mutex_unlock(&mi->data_lock); } else if (check_io_slave_killed(mi, NULL)) @@ -3187,6 +3189,14 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, else { idle= mi->rli.sql_thread_caught_up; + + /* + The idleness of the SQL thread is needed for the parallel slave + because events can be ignored before distribution to a worker thread. + That is, Seconds_Behind_Master should still be calculated and visible + while the slave is processing ignored events, such as those skipped + due to slave_skip_counter. + */ if (mi->using_parallel() && idle && !mi->rli.parallel.workers_idle()) idle= false; } @@ -4146,7 +4156,6 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, thd, STRING_WITH_LEN( "now SIGNAL paused_on_event WAIT_FOR sql_thread_continue"))); - DBUG_SET("-d,pause_sql_thread_on_next_event"); mysql_mutex_lock(&rli->data_lock); }); @@ -4163,7 +4172,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, the user might be surprised to see a claim that the slave is up to date long before those queued events are actually executed. */ - if ((!rli->mi->using_parallel()) && event_can_update_last_master_timestamp(ev)) + if ((!rli->mi->using_parallel()) && + event_can_update_last_master_timestamp(ev)) { rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; rli->sql_thread_caught_up= false; @@ -4218,9 +4228,22 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, if (rli->mi->using_parallel()) { - if (unlikely((rli->last_master_timestamp == 0 || - rli->sql_thread_caught_up) && - event_can_update_last_master_timestamp(ev))) + /* + rli->sql_thread_caught_up is checked and negated here to ensure that + the value of Seconds_Behind_Master in SHOW SLAVE STATUS is consistent + with the update of last_master_timestamp. It was previously unset + immediately after reading an event from the relay log; however, for the + duration between that unset and the time that LMT would be updated + could lead to spikes in SBM. + + The check for queued_count == dequeued_count ensures the worker threads + are all idle (i.e. all events have been executed). + */ + if ((unlikely(rli->last_master_timestamp == 0) || + (rli->sql_thread_caught_up && + (rli->last_inuse_relaylog->queued_count == + rli->last_inuse_relaylog->dequeued_count))) && + event_can_update_last_master_timestamp(ev)) { if (rli->last_master_timestamp < ev->when) { From 2a46b358a768f3af837386585d2f3d6720644ed8 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 25 Jul 2023 13:23:18 +0200 Subject: [PATCH 43/43] new WolfSSL v5.6.3-stable --- extra/wolfssl/CMakeLists.txt | 4 +++- extra/wolfssl/wolfssl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extra/wolfssl/CMakeLists.txt b/extra/wolfssl/CMakeLists.txt index 0aee5865867..826587df242 100644 --- a/extra/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/CMakeLists.txt @@ -147,7 +147,9 @@ IF(WOLFSSL_X86_64_BUILD) LIST(APPEND WOLFCRYPT_SOURCES ${WOLFCRYPT_SRCDIR}/cpuid.c) IF(MSVC) SET(WOLFSSL_AESNI 1) - LIST(APPEND WOLFCRYPT_SOURCES ${WOLFCRYPT_SRCDIR}/aes_asm.asm) + LIST(APPEND WOLFCRYPT_SOURCES + ${WOLFCRYPT_SRCDIR}/aes_asm.asm + ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.asm) IF(CMAKE_C_COMPILER_ID MATCHES Clang) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -msse4.2 -mpclmul -mrdrnd -mrdseed") ENDIF() diff --git a/extra/wolfssl/wolfssl b/extra/wolfssl/wolfssl index 4fbd4fd36a2..3b3c175af0e 160000 --- a/extra/wolfssl/wolfssl +++ b/extra/wolfssl/wolfssl @@ -1 +1 @@ -Subproject commit 4fbd4fd36a21efd9d1a7e17aba390e91c78693b1 +Subproject commit 3b3c175af0e993ffaae251871421e206cc41963f