From e938d7c18f6f117b63fcfc50dd975029d92409a7 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 28 Aug 2023 19:02:23 +0530 Subject: [PATCH 1/4] MDEV-32028 InnoDB scrubbing doesn't write zero while freeing the extent Problem: ======== InnoDB fails to mark the page status as FREED during freeing of an extent of a segment. This behaviour affects scrubbing and doesn't write all zeroes in file even though pages are freed. Solution: ======== InnoDB should mark the page status as FREED before reinitialize the extent descriptor entry. --- mysql-test/suite/innodb/r/scrub_debug.result | 22 ++++++++++++++ mysql-test/suite/innodb/t/scrub_debug.test | 31 ++++++++++++++++++++ storage/innobase/fsp/fsp0fsp.cc | 4 +-- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/innodb/r/scrub_debug.result create mode 100644 mysql-test/suite/innodb/t/scrub_debug.test diff --git a/mysql-test/suite/innodb/r/scrub_debug.result b/mysql-test/suite/innodb/r/scrub_debug.result new file mode 100644 index 00000000000..5fbf1250b21 --- /dev/null +++ b/mysql-test/suite/innodb/r/scrub_debug.result @@ -0,0 +1,22 @@ +SET @save_debug=@@GLOBAL.INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG; +SET @save_scrub=@@GLOBAL.INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED; +SET @save_freq=@@GLOBAL.INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY; +SET GLOBAL INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY=1; +SET GLOBAL INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED=1; +SET GLOBAL INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG=2; +CREATE TABLE t1(f1 INT AUTO_INCREMENT PRIMARY KEY, +f2 VARCHAR(256) GENERATED ALWAYS as('repairman'), +INDEX idx(f2))ENGINE= InnoDB; +INSERT INTO t1(f1) SELECT seq FROM seq_1_to_50; +FLUSH TABLE t1 FOR EXPORT; +FOUND 108 /repairman/ in t1.ibd +UNLOCK TABLES; +ALTER TABLE t1 DROP INDEX idx; +InnoDB 0 transactions not purged +FLUSH TABLE t1 FOR EXPORT; +NOT FOUND /repairman/ in t1.ibd +UNLOCK TABLES; +DROP TABLE t1; +SET GLOBAL INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG=@save_debug; +SET GLOBAL INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED=@save_scrub; +SET GLOBAL INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY = @save_freq; diff --git a/mysql-test/suite/innodb/t/scrub_debug.test b/mysql-test/suite/innodb/t/scrub_debug.test new file mode 100644 index 00000000000..141b0f0c5ba --- /dev/null +++ b/mysql-test/suite/innodb/t/scrub_debug.test @@ -0,0 +1,31 @@ +--source include/have_innodb.inc +--source include/have_sequence.inc +--source include/have_debug.inc + +SET @save_debug=@@GLOBAL.INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG; +SET @save_scrub=@@GLOBAL.INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED; +SET @save_freq=@@GLOBAL.INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY; + +SET GLOBAL INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY=1; +SET GLOBAL INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED=1; +SET GLOBAL INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG=2; +let $MYSQLD_DATADIR=`select @@datadir`; +CREATE TABLE t1(f1 INT AUTO_INCREMENT PRIMARY KEY, + f2 VARCHAR(256) GENERATED ALWAYS as('repairman'), + INDEX idx(f2))ENGINE= InnoDB; +INSERT INTO t1(f1) SELECT seq FROM seq_1_to_50; +FLUSH TABLE t1 FOR EXPORT; +let SEARCH_PATTERN= repairman; +let SEARCH_FILE= $MYSQLD_DATADIR/test/t1.ibd; +-- source include/search_pattern_in_file.inc +UNLOCK TABLES; + +ALTER TABLE t1 DROP INDEX idx; +-- source include/wait_all_purged.inc +FLUSH TABLE t1 FOR EXPORT; +-- source include/search_pattern_in_file.inc +UNLOCK TABLES; +DROP TABLE t1; +SET GLOBAL INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG=@save_debug; +SET GLOBAL INNODB_IMMEDIATE_SCRUB_DATA_UNCOMPRESSED=@save_scrub; +SET GLOBAL INNODB_PURGE_RSEG_TRUNCATE_FREQUENCY = @save_freq; diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 7532ce85b6b..64ca0457313 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -2683,14 +2683,14 @@ fseg_free_extent( not_full_n_used - descr_n_used); } - fsp_free_extent(space, page, mtr); - for (uint32_t i = 0; i < FSP_EXTENT_SIZE; i++) { if (!xdes_is_free(descr, i)) { buf_page_free(space, first_page_in_extent + i, mtr, __FILE__, __LINE__); } } + + fsp_free_extent(space, page, mtr); } /**********************************************************************//** From 53499cd1ea1c8092460924224d78a286d617492d Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 29 Aug 2023 11:51:01 +0400 Subject: [PATCH 2/4] MDEV-31303 Key not used when IN clause has both signed and usigned values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: This patch enables possible index optimization when the WHERE clause has an IN condition of the form: signed_or_unsigned_column IN (signed_or_unsigned_constant, signed_or_unsigned_constant [,signed_or_unsigned_constant]*) when the IN list constants are of different signess, e.g.: WHERE signed_column IN (signed_constant, unsigned_constant ...) WHERE unsigned_column IN (signed_constant, unsigned_constant ...) Details: In a condition like: WHERE unsigned_predicant IN (1, LONGLONG_MAX + 1) comparison handlers for individual (predicant,value) pairs are calculated as follows: * unsigned_predicant and 1 produce &type_handler_newdecimal * unsigned_predicant and (LONGLONG_MAX + 1) produce &type_handler_slonglong The old code decided that it could not use bisection because the two pairs had different comparison handlers. As a result, bisection was not allowed, and, in case of an indexed integer column predicant the index on the column was not used. The new code catches special cases like: signed_predicant IN (signed_constant, unsigned_constant) unsigned_predicant IN (signed_constant, unsigned_constant) It enables bisection using in_longlong, which supports a mixture of predicant and values of different signess. In case when the predicant is an indexed column this change automatically enables index range optimization. Thanks to Vicențiu Ciorbaru for proposing the idea and for preparing MTR tests. --- mysql-test/main/func_debug.result | 9 +++++--- mysql-test/main/func_in.result | 37 +++++++++++++++++++++++++++++++ mysql-test/main/func_in.test | 34 ++++++++++++++++++++++++++++ sql/item_cmpfunc.cc | 36 ++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/func_debug.result b/mysql-test/main/func_debug.result index c8efcf09d41..37f2a19fc6c 100644 --- a/mysql-test/main/func_debug.result +++ b/mysql-test/main/func_debug.result @@ -912,7 +912,8 @@ a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED)) Warnings: Note 1105 DBUG: [0] arg=1 handler=0 (bigint) Note 1105 DBUG: [1] arg=2 handler=1 (decimal) -Note 1105 DBUG: types_compatible=no bisect=no +Note 1105 DBUG: found a mix of UINT and SINT +Note 1105 DBUG: types_compatible=yes bisect=yes SELECT a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) FROM t1; a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) Warnings: @@ -950,7 +951,8 @@ a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED)) Warnings: Note 1105 DBUG: [0] arg=1 handler=0 (bigint) Note 1105 DBUG: [1] arg=2 handler=1 (decimal) -Note 1105 DBUG: types_compatible=no bisect=no +Note 1105 DBUG: found a mix of UINT and SINT +Note 1105 DBUG: types_compatible=yes bisect=yes SELECT a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) FROM t1; a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) Warnings: @@ -1624,7 +1626,8 @@ a b Warnings: Note 1105 DBUG: [0] arg=1 handler=0 (bigint) Note 1105 DBUG: [1] arg=2 handler=1 (decimal) -Note 1105 DBUG: types_compatible=no bisect=no +Note 1105 DBUG: found a mix of UINT and SINT +Note 1105 DBUG: types_compatible=yes bisect=no DROP TABLE t1; # # MDEV-11554 Wrong result for CASE on a mixture of signed and unsigned expressions diff --git a/mysql-test/main/func_in.result b/mysql-test/main/func_in.result index dd7125c393a..e81ececf752 100644 --- a/mysql-test/main/func_in.result +++ b/mysql-test/main/func_in.result @@ -944,3 +944,40 @@ Warning 1292 Truncated incorrect DECIMAL value: '0x' # # End of 10.4 tests # +# +# Start of 10.5 tests +# +# +# MDEV-31303: Key not used +# +CREATE TABLE `a` ( +`id` bigint AUTO_INCREMENT PRIMARY KEY, +`c1` bigint unsigned, +KEY (`c1`) +); +INSERT INTO `a` VALUES (1,9223382399205928659),(2,9223384207280813348), +(3,9223385953115437234),(4,9223387250780556749),(5,9223387354282558788), +(6,9223387603870501596),(7,9223389270813433667),(8,9223389903231468827), +(9,9223390280789586779),(10,9223391591398222899),(11,9223391875473564350), +(12,9223393152250049433),(13,9223393939696790223),(14,9223394417225350415), +(15,9223397646397141015),(16,9223398025879291243),(17,9223399038671098072), +(18,9223399534968874556),(19,9223400449518009285),(20,9223400860292643549), +(21,9223400940692256924),(22,9223401073791948119),(23,9223402820804649616), +(24,9223403470951992681),(25,9223405581879567267),(26,9223405754978563829), +(27,9223405972966828221), (28, 9223372036854775808), (29, 9223372036854775807) ; +explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 ); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE a range c1 c1 9 NULL 2 Using where; Using index +explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 ); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE a range c1 c1 9 NULL 2 Using where; Using index +SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 ); +c1 +9223372036854775807 +SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 ); +c1 +9223372036854775808 +drop table `a`; +# +# End of 10.5 tests +# diff --git a/mysql-test/main/func_in.test b/mysql-test/main/func_in.test index 2581a07ac09..6cddf1dea5c 100644 --- a/mysql-test/main/func_in.test +++ b/mysql-test/main/func_in.test @@ -726,3 +726,37 @@ SELECT ('0x',1) IN ((0,1),(1,1)); --echo # End of 10.4 tests --echo # +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MDEV-31303: Key not used +--echo # +CREATE TABLE `a` ( + `id` bigint AUTO_INCREMENT PRIMARY KEY, + `c1` bigint unsigned, + KEY (`c1`) +); + +INSERT INTO `a` VALUES (1,9223382399205928659),(2,9223384207280813348), +(3,9223385953115437234),(4,9223387250780556749),(5,9223387354282558788), +(6,9223387603870501596),(7,9223389270813433667),(8,9223389903231468827), +(9,9223390280789586779),(10,9223391591398222899),(11,9223391875473564350), +(12,9223393152250049433),(13,9223393939696790223),(14,9223394417225350415), +(15,9223397646397141015),(16,9223398025879291243),(17,9223399038671098072), +(18,9223399534968874556),(19,9223400449518009285),(20,9223400860292643549), +(21,9223400940692256924),(22,9223401073791948119),(23,9223402820804649616), +(24,9223403470951992681),(25,9223405581879567267),(26,9223405754978563829), +(27,9223405972966828221), (28, 9223372036854775808), (29, 9223372036854775807) ; + +explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 ); +explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 ); +SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 ); +SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 ); +drop table `a`; + +--echo # +--echo # End of 10.5 tests +--echo # + diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 9903bb60d05..4a2de58e748 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4442,6 +4442,42 @@ bool Item_func_in::fix_length_and_dec() return TRUE; } + if (!arg_types_compatible && comparator_count() == 2) + { + /* + Catch a special case: a mixture of signed and unsigned integer types. + in_longlong can handle such cases. + + Note, prepare_predicant_and_values() aggregates this mixture as follows: + - signed+unsigned produce &type_handler_newdecimal. + - signed+signed or unsigned+unsigned produce &type_handler_slonglong + So we have extactly two distinct handlers. + + The code below assumes that unsigned longlong is handled + by &type_handler_slonglong in comparison context, + which may change in the future to &type_handler_ulonglong. + The DBUG_ASSERT is needed to address this change here properly. + */ + DBUG_ASSERT(type_handler_ulonglong.type_handler_for_comparison() == + &type_handler_slonglong); + // Let's check if all arguments are of integer types + uint found_int_args= 0; + for (uint i= 0; i < arg_count; i++, found_int_args++) + { + if (args[i]->type_handler_for_comparison() != &type_handler_slonglong) + break; + } + if (found_int_args == arg_count) + { + // All arguments are integers. Switch to integer comparison. + arg_types_compatible= true; + DBUG_EXECUTE_IF("Item_func_in", + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_UNKNOWN_ERROR, "DBUG: found a mix of UINT and SINT");); + m_comparator.set_handler(&type_handler_slonglong); + } + } + if (arg_types_compatible) // Bisection condition #1 { if (m_comparator.type_handler()-> From 3c86765efef97862ab3ea8021ec9348ea995e915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 31 Aug 2023 12:08:40 +0300 Subject: [PATCH 3/4] MDEV-23974 fixup: Use standard quotes in have_innodb.inc This fixes the following test: set sql_mode=ORACLE; --source include/have_innodb.inc --- mysql-test/include/have_innodb.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/include/have_innodb.inc b/mysql-test/include/have_innodb.inc index 0de070e1994..8c9cdb54363 100644 --- a/mysql-test/include/have_innodb.inc +++ b/mysql-test/include/have_innodb.inc @@ -3,7 +3,7 @@ # will be skipped unless innodb is enabled # --disable_query_log -if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "MSAN%"`) +if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like 'MSAN%'`) { SET STATEMENT sql_log_bin=0 FOR call mtr.add_suppression("InnoDB: Trying to delete tablespace.*pending operations"); From 2db5f1b2985d2c921e73d19561332246fa43557e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 31 Aug 2023 12:14:49 +0300 Subject: [PATCH 4/4] MDEV-32049 Deadlock due to log_free_check() in trx_purge_truncate_history() The function log_free_check() is not supposed to be invoked while the caller is holding any InnoDB synchronization objects, such as buffer page latches, tablespace latches, index tree latches, or in this case, rseg->mutex (rseg->latch in 10.6 or later). A hang was reported in 10.6 where several threads were waiting for an rseg->latch that had been exclusively acquired in trx_purge_truncate_history(), which invoked log_free_check() inside trx_purge_truncate_rseg_history(). Because the threads that were waiting for the rseg->latch were holding exclusive latches on some index pages, log_free_check() was unable to advance the checkpoint because those index pages could not be written out. trx_purge_truncate_history(): Invoke log_free_check() before acquiring the rseg->mutex and invoking trx_purge_free_segment(). trx_purge_free_segment(): Do not invoke log_free_check() in order to avoid a deadlock. --- storage/innobase/trx/trx0purge.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index c6adaf5f2bf..a297ca4921e 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -346,7 +346,6 @@ static void trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) { mtr.commit(); - log_free_check(); mtr.start(); ut_ad(mutex_own(&rseg->mutex)); @@ -376,7 +375,6 @@ void trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) This does not matter when using multiple innodb_undo_tablespaces; innodb_undo_log_truncate=ON will be able to reclaim the space. */ - log_free_check(); mtr.start(); ut_ad(rw_lock_s_lock_nowait(block->debug_latch, __FILE__, __LINE__)); rw_lock_x_lock(&block->lock); @@ -545,6 +543,7 @@ void trx_purge_truncate_history() { ut_ad(rseg->id == i); ut_ad(rseg->is_persistent()); + log_free_check(); mutex_enter(&rseg->mutex); trx_purge_truncate_rseg_history(*rseg, head, !rseg->trx_ref_count &&