diff --git a/COMMUNITY_CONTRIBUTIONS.md b/COMMUNITY_CONTRIBUTIONS.md index 39776384bef..b306b681d51 100644 --- a/COMMUNITY_CONTRIBUTIONS.md +++ b/COMMUNITY_CONTRIBUTIONS.md @@ -1,4 +1,4 @@ -# Community Contributions Process # +# Community Contributions Process MariaDB Server is an Open Source project that thrives thanks to its community. MariaDB Foundation handles all community contributions in a timely manner. @@ -25,7 +25,7 @@ But it's important to always have both prior to review. There are constant efforts (including background processes) to make sure the states of the Jira and the corresponding pull request(s) are kept in sync. -### What should go into the jira ### +### What should go into the jira The [Jira](https://jira.mariadb.org) is used to describe the design of the feature: why is it a good idea, what should work, what might not work etc. @@ -45,9 +45,9 @@ See [here](https://mariadb.org/contribute/#contribute-to-testing) > [!CAUTION] > If the jira already exists and is assigned to somebody please reach out -> to them **before** starting to working on it to avoid effort duplication. +> to them **before** starting to working on it to avoid effort duplication. -### What should go into the pull request ### +### What should go into the pull request The pull request is used to describe the implementation. E.g. how the bug is fixed. Or how the feature is implemented in detail. @@ -64,7 +64,7 @@ And that commit should have a commit message that's compliant with > If there's no Jira or no reference to it in your pull request > there might be delays in processing it -## States of a Community Contribution ## +## States of a Community Contribution ```mermaid stateDiagram-v2 @@ -87,7 +87,7 @@ stateDiagram-v2 Closed --> [*] ``` -### Draft ### +### Draft * Pull request * State: Draft @@ -114,7 +114,7 @@ the GitHub pull request viewer UI. > and "External Contribution" pull request labels and there's > been no activity for 3 months. -### Open ## +### Open * Pull request * State: Open @@ -169,7 +169,7 @@ Questions and Answers > label to pull requests in this state, if absent. > This marks the contribution entering the preliminary review queue. -### Preliminary Review ### +### Preliminary Review * Pull request * State: Open @@ -199,7 +199,7 @@ The preliminary reviewer does the following: Or that an internal developer has taken over the contribution to re-do it and push it directly. -### Final review ### +### Final review * Pull request * State: Open @@ -245,7 +245,7 @@ detailed information: > There's a requirement that certain contributions need to undergo > a separate test step. This is usually done for the new features or > for more complex and risky bug fixes. If that's the desired state, -> a test engineer will be assigned as a final reviewer. See +> a test engineer will be assigned as a final reviewer. See > https://mariadb.com/docs/server/reference/product-development/mariadb-quality-development-rules > for more details. @@ -256,7 +256,7 @@ detailed information: > in the middle of the new feature development cycle or vice versa > it might take up to a couple of months for a developer to do a review. -### Approved ### +### Approved * Pull request * State: Open @@ -280,7 +280,7 @@ The person that does the merge might need some assistance by the contributor. When the pull request is successfully merged the Jira should be moved to "Closed" state and its target version should be specified. -### Merged ### +### Merged * Pull request * State: Merged @@ -295,7 +295,7 @@ This is the "success" final state for a contribution. Ideally all contributions will end up in this state. No further actions are necessary. -### Closed ### +### Closed * Pull request * State: Closed diff --git a/client/mysql.cc b/client/mysql.cc index 62bd07a7679..a3fef318f7f 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1415,6 +1415,9 @@ int main(int argc,char *argv[]) mysql_thread_id(&mysql), server_version_string(&mysql)); put_info((char*) glob_buffer.ptr(),INFO_INFO); put_info(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"), INFO_INFO); + put_info("Help others discover MariaDB." + " Star it on GitHub: https://github.com/MariaDB/server\n", + INFO_INFO); } #ifdef HAVE_READLINE diff --git a/cmake/build_depends.cmake b/cmake/build_depends.cmake index 758ba184628..d01019074fa 100644 --- a/cmake/build_depends.cmake +++ b/cmake/build_depends.cmake @@ -28,6 +28,10 @@ IF(RPM) STRING(REGEX REPLACE "^Have library " "" L ${H}) SET(V ${L}_LIBRARY) FIND_LIBRARY(${V} ${L}) + ELSEIF(H MATCHES "^Have includes? [^/]" AND ${V}) + STRING(REGEX REPLACE "^Have includes? (.*;)?" "" I ${H}) + SET(V ${I}_INCLUDE) + FIND_FILE(${V} ${I}) ENDIF() GET_PROPERTY(T CACHE ${V} PROPERTY TYPE) IF ((T STREQUAL FILEPATH OR V MATCHES "^CMAKE_COMMAND$") AND ${V} MATCHES "^/") diff --git a/cmake/dtrace.cmake b/cmake/dtrace.cmake index 0ba7fc7d6dd..cd209f067e0 100644 --- a/cmake/dtrace.cmake +++ b/cmake/dtrace.cmake @@ -59,6 +59,11 @@ MACRO(CHECK_DTRACE) SET(DTRACE_FLAGS -64 CACHE INTERNAL "DTrace architecture flags") ENDIF() ENDIF() + include(CheckIncludeFile) + CHECK_INCLUDE_FILE("sys/sdt.h" HAVE_SYS_SDT_H) + IF(NOT HAVE_SYS_SDT_H) + SET(ENABLE_DTRACE OFF CACHE BOOL "Disable dtrace") + ENDIF() ENDMACRO() CHECK_DTRACE() diff --git a/mysql-test/include/galera_wait_ready.inc b/mysql-test/include/galera_wait_ready.inc index a726116f000..83e8717bb11 100644 --- a/mysql-test/include/galera_wait_ready.inc +++ b/mysql-test/include/galera_wait_ready.inc @@ -2,11 +2,26 @@ # # Waits for galera node to transition to READY state. # +# Parameters +# ---------- +# $galera_wait_ready_timeout (optional) +# Seconds to wait before giving up, 60 by default. Raise it around a +# restart which has to wait for a full SST, as that takes considerably +# longer than a plain restart and on a loaded machine can exceed the +# default. The value is reset here, so it only applies to the next +# sourcing of this file. +# --enable_reconnect --disable_query_log --disable_result_log + let $wait_counter = 600; +if ($galera_wait_ready_timeout) +{ + let $wait_counter = $($galera_wait_ready_timeout * 10); +} + while ($wait_counter) { --disable_abort_on_error @@ -25,7 +40,7 @@ while ($wait_counter) if (!$success) { - die "Server did not transition to READY state"; + die "Server did not transition to READY state in $_galera_wait_ready_timeout seconds"; } --disable_reconnect --enable_query_log diff --git a/mysql-test/main/json_normalize.result b/mysql-test/main/json_normalize.result index cd5b6d3d2bf..ee92e047683 100644 --- a/mysql-test/main/json_normalize.result +++ b/mysql-test/main/json_normalize.result @@ -45,6 +45,21 @@ json_normalize('{ "invalid": "no_close"') NULL Warnings: Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +select json_normalize(''); +json_normalize('') +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +select json_normalize(' '); +json_normalize(' ') +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +select json_normalize('123 garbage'); +json_normalize('123 garbage') +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_normalize' at position 5 drop table t1; drop view v1; create table t1 (text varchar(200) character set 'latin1'); diff --git a/mysql-test/main/json_normalize.test b/mysql-test/main/json_normalize.test index a1c0b12026d..d4f27efd2c8 100644 --- a/mysql-test/main/json_normalize.test +++ b/mysql-test/main/json_normalize.test @@ -25,6 +25,9 @@ order by json; select json_normalize(NULL); select json_normalize('{ "invalid": "no_close"'); +select json_normalize(''); +select json_normalize(' '); +select json_normalize('123 garbage'); drop table t1; drop view v1; diff --git a/mysql-test/main/mdev_40698.result b/mysql-test/main/mdev_40698.result new file mode 100644 index 00000000000..3c51bf2d37d --- /dev/null +++ b/mysql-test/main/mdev_40698.result @@ -0,0 +1,54 @@ +# +# MDEV-40698: ROLLUP query returns incorrect result with empty result set +# +CREATE TABLE t0(a INT, b INT, c INT); +SELECT *, COUNT(0) FROM t0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT *, COUNT(0) FROM t0 WHERE 1=0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT *, COUNT(0), ROW_NUMBER() OVER () FROM t0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) ROW_NUMBER() OVER () +NULL NULL NULL 0 1 +SELECT 1 IN (SELECT a FROM t0 WHERE a>0 GROUP BY a WITH ROLLUP); +1 IN (SELECT a FROM t0 WHERE a>0 GROUP BY a WITH ROLLUP) +NULL +INSERT INTO t0 VALUES (0, 0, 0),(0, 0, 0); +SELECT *, COUNT(0) FROM t0 WHERE a > 0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT EXISTS (SELECT t0.a FROM t0 x WHERE 1=0 GROUP BY x.a WITH ROLLUP) AS e +FROM t0; +e +1 +1 +SELECT DISTINCT a FROM t0 GROUP BY a, b WITH ROLLUP; +a +0 +NULL +CREATE TABLE t1 (a INT, b INT, c INT) ENGINE=InnoDB; +SELECT *, COUNT(0) FROM t1 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT *, COUNT(0) FROM t1 WHERE 1=0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT *, COUNT(0), ROW_NUMBER() OVER () FROM t1 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) ROW_NUMBER() OVER () +NULL NULL NULL 0 1 +INSERT INTO t1 VALUES (0, 0, 0),(0, 0, 0); +SELECT *, COUNT(0) FROM t1 WHERE a > 0 GROUP BY a, b, c WITH ROLLUP; +a b c COUNT(0) +NULL NULL NULL 0 +SELECT EXISTS (SELECT t1.a FROM t1 x WHERE 1=0 GROUP BY x.a WITH ROLLUP) AS e +FROM t1; +e +1 +1 +SELECT DISTINCT a FROM t1 GROUP BY a, b WITH ROLLUP; +a +0 +NULL +DROP TABLE t0, t1; +# End of 10.11 tests diff --git a/mysql-test/main/mdev_40698.test b/mysql-test/main/mdev_40698.test new file mode 100644 index 00000000000..dfe7a0fe68b --- /dev/null +++ b/mysql-test/main/mdev_40698.test @@ -0,0 +1,30 @@ +--source include/have_innodb.inc + +--echo # +--echo # MDEV-40698: ROLLUP query returns incorrect result with empty result set +--echo # + +CREATE TABLE t0(a INT, b INT, c INT); +SELECT *, COUNT(0) FROM t0 GROUP BY a, b, c WITH ROLLUP; +SELECT *, COUNT(0) FROM t0 WHERE 1=0 GROUP BY a, b, c WITH ROLLUP; +SELECT *, COUNT(0), ROW_NUMBER() OVER () FROM t0 GROUP BY a, b, c WITH ROLLUP; +SELECT 1 IN (SELECT a FROM t0 WHERE a>0 GROUP BY a WITH ROLLUP); +INSERT INTO t0 VALUES (0, 0, 0),(0, 0, 0); +SELECT *, COUNT(0) FROM t0 WHERE a > 0 GROUP BY a, b, c WITH ROLLUP; +SELECT EXISTS (SELECT t0.a FROM t0 x WHERE 1=0 GROUP BY x.a WITH ROLLUP) AS e + FROM t0; +SELECT DISTINCT a FROM t0 GROUP BY a, b WITH ROLLUP; + +CREATE TABLE t1 (a INT, b INT, c INT) ENGINE=InnoDB; +SELECT *, COUNT(0) FROM t1 GROUP BY a, b, c WITH ROLLUP; +SELECT *, COUNT(0) FROM t1 WHERE 1=0 GROUP BY a, b, c WITH ROLLUP; +SELECT *, COUNT(0), ROW_NUMBER() OVER () FROM t1 GROUP BY a, b, c WITH ROLLUP; +INSERT INTO t1 VALUES (0, 0, 0),(0, 0, 0); +SELECT *, COUNT(0) FROM t1 WHERE a > 0 GROUP BY a, b, c WITH ROLLUP; +SELECT EXISTS (SELECT t1.a FROM t1 x WHERE 1=0 GROUP BY x.a WITH ROLLUP) AS e + FROM t1; +SELECT DISTINCT a FROM t1 GROUP BY a, b WITH ROLLUP; + +DROP TABLE t0, t1; + +--echo # End of 10.11 tests diff --git a/mysql-test/main/mysql-interactive.result b/mysql-test/main/mysql-interactive.result index 0eb3a53cac7..cd083339ace 100644 --- a/mysql-test/main/mysql-interactive.result +++ b/mysql-test/main/mysql-interactive.result @@ -10,6 +10,8 @@ Your MariaDB connection id is X Server version: Y Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. +Help others discover MariaDB. Star it on GitHub: https://github.com/MariaDB/server + Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> delimiter $ @@ -40,6 +42,8 @@ Your MariaDB connection id is X Server version: Y Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. +Help others discover MariaDB. Star it on GitHub: https://github.com/MariaDB/server + Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> create database db1; diff --git a/mysql-test/main/parser.result b/mysql-test/main/parser.result index 32cc909dbeb..b8ebd051a44 100644 --- a/mysql-test/main/parser.result +++ b/mysql-test/main/parser.result @@ -2272,4 +2272,51 @@ select 1b from test.1a; select select.1c from test.select; 1c drop table 1a, `select`; +# +# MDEV-37167 Nested BEGINs (4600+) cause a segmentation fault +# +# +# Unclosed BEGINs +# +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1 +# +# Well-formed deep routine +# +DROP PROCEDURE p2; +# +# Same, with variables, so the contexts are not empty +# +DROP PROCEDURE p3; +# +# A deep FUNCTION tears down the same way +# +DROP FUNCTION f1; +# +# Invoking a deep routine must not crash. It is rejected with +# ER_STACK_OVERRUN_NEED_MORE when the frame does not fit the thread +# stack, which depends on the build, so both outcomes are accepted. +# +CALL p4(); +DROP PROCEDURE p4; +# +# Nested scopes must still resolve correctly +# +CREATE PROCEDURE p5() +BEGIN +DECLARE a INT DEFAULT 1; +BEGIN +DECLARE b INT DEFAULT 2; +BEGIN +DECLARE c INT DEFAULT 3; +SELECT a + b + c AS total; +END; +END; +END$$ +CALL p5(); +total +6 +DROP PROCEDURE p5; +SELECT 'Server is alive' AS state; +state +Server is alive # End of 10.11 tests diff --git a/mysql-test/main/parser.test b/mysql-test/main/parser.test index 138234a4d25..0fd2d5e63df 100644 --- a/mysql-test/main/parser.test +++ b/mysql-test/main/parser.test @@ -2078,4 +2078,78 @@ select select.1c from test.select; drop table 1a, `select`; +--echo # +--echo # MDEV-37167 Nested BEGINs (4600+) cause a segmentation fault +--echo # + +# Each nested BEGIN adds one sp_pcontext. Freeing them used to recurse once +# per nesting level and overran the thread stack. Building the run-time frame +# still recurses, but checks the stack now, and the depth where it gives up +# follows thread_stack, so the invoking statements accept either outcome. + +--echo # +--echo # Unclosed BEGINs +--echo # +--disable_query_log +--error ER_PARSE_ERROR +--eval CREATE PROCEDURE p1() $(repeat("BEGIN ", 6000)) +--enable_query_log + +--echo # +--echo # Well-formed deep routine +--echo # +--disable_query_log +--eval CREATE PROCEDURE p2() $(repeat("BEGIN ", 6000)) $(repeat("END; ", 5999)) END +--enable_query_log +DROP PROCEDURE p2; + +--echo # +--echo # Same, with variables, so the contexts are not empty +--echo # +--disable_query_log +--eval CREATE PROCEDURE p3() $(repeat("BEGIN DECLARE v INT DEFAULT 1; ", 2000)) $(repeat("END; ", 1999)) END +--enable_query_log +DROP PROCEDURE p3; + +--echo # +--echo # A deep FUNCTION tears down the same way +--echo # +--disable_query_log +--eval CREATE FUNCTION f1() RETURNS INT $(repeat("BEGIN ", 6000)) RETURN 1; $(repeat("END; ", 5999)) END +--enable_query_log +DROP FUNCTION f1; + +--echo # +--echo # Invoking a deep routine must not crash. It is rejected with +--echo # ER_STACK_OVERRUN_NEED_MORE when the frame does not fit the thread +--echo # stack, which depends on the build, so both outcomes are accepted. +--echo # +--disable_query_log +--eval CREATE PROCEDURE p4() $(repeat("BEGIN ", 6000)) $(repeat("END; ", 5999)) END +--enable_query_log +--error 0,ER_STACK_OVERRUN_NEED_MORE +CALL p4(); +DROP PROCEDURE p4; + +--echo # +--echo # Nested scopes must still resolve correctly +--echo # +DELIMITER $$; +CREATE PROCEDURE p5() +BEGIN + DECLARE a INT DEFAULT 1; + BEGIN + DECLARE b INT DEFAULT 2; + BEGIN + DECLARE c INT DEFAULT 3; + SELECT a + b + c AS total; + END; + END; +END$$ +DELIMITER ;$$ +CALL p5(); +DROP PROCEDURE p5; + +SELECT 'Server is alive' AS state; + --echo # End of 10.11 tests diff --git a/mysql-test/main/ps_error.result b/mysql-test/main/ps_error.result index ad178f86915..f3fa78a1bbe 100644 --- a/mysql-test/main/ps_error.result +++ b/mysql-test/main/ps_error.result @@ -10,6 +10,7 @@ EXECUTE stmt; ERROR 22007: Truncated incorrect DOUBLE value: 'foo' SELECT a FROM t1 GROUP BY NULL WITH ROLLUP; a +NULL DROP TABLE t1; SET sql_mode=DEFAULT; SET SQL_MODE= 'STRICT_ALL_TABLES'; diff --git a/mysql-test/main/win.result b/mysql-test/main/win.result index 78f39de208c..7af4a87d3eb 100644 --- a/mysql-test/main/win.result +++ b/mysql-test/main/win.result @@ -3539,6 +3539,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables select distinct 1, row_number() over (order by 1) from t1 where a=0 group by a with rollup; 1 row_number() over (order by 1) +1 1 drop table t1; explain SELECT DISTINCT BIT_OR(100) OVER () FROM dual diff --git a/mysql-test/suite/binlog/r/binlog_autocommit_off_no_hang.result b/mysql-test/suite/binlog/r/binlog_autocommit_off_no_hang.result index 71eecd881ca..8adeb4827ce 100644 --- a/mysql-test/suite/binlog/r/binlog_autocommit_off_no_hang.result +++ b/mysql-test/suite/binlog/r/binlog_autocommit_off_no_hang.result @@ -3,4 +3,11 @@ ALTER TABLE mysql.gtid_slave_pos ENGINE=innodb; # Set gtid_slave_pos should not hang SET GLOBAL gtid_slave_pos=@@gtid_binlog_pos; COMMIT; +### MDEV-22848: SET GLOBAL gtid_slave_pos leaves dangling partial transaction +CREATE TABLE t1 (c INT) ENGINE=MyISAM; +SET GLOBAL gtid_slave_pos="0-1-100"; +INSERT INTO t1 VALUES (0); +DROP TABLE not_there; +ERROR 42S02: Unknown table 'test.not_there' +DROP TABLE t1; RESET MASTER; diff --git a/mysql-test/suite/binlog/r/binlog_base64_flag.result b/mysql-test/suite/binlog/r/binlog_base64_flag.result index 4b75b712aee..74339e16d03 100644 --- a/mysql-test/suite/binlog/r/binlog_base64_flag.result +++ b/mysql-test/suite/binlog/r/binlog_base64_flag.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("BINLOG_BASE64_EVENT: According to the master's version"); +call mtr.add_suppression("BINLOG_BASE64_EVENT: Master's version is earlier than"); call mtr.add_suppression("BINLOG_BASE64_EVENT: Column 1 of table 'test.char128_utf8' cannot be converted"); CREATE TABLE t1 (a int); INSERT INTO t1 VALUES (1), (1); diff --git a/mysql-test/suite/binlog/t/binlog_autocommit_off_no_hang.test b/mysql-test/suite/binlog/t/binlog_autocommit_off_no_hang.test index 0038a6e2bf7..2c1945f84f5 100644 --- a/mysql-test/suite/binlog/t/binlog_autocommit_off_no_hang.test +++ b/mysql-test/suite/binlog/t/binlog_autocommit_off_no_hang.test @@ -37,6 +37,15 @@ ALTER TABLE mysql.gtid_slave_pos ENGINE=innodb; SET GLOBAL gtid_slave_pos=@@gtid_binlog_pos; COMMIT; +--echo ### MDEV-22848: SET GLOBAL gtid_slave_pos leaves dangling partial transaction +CREATE TABLE t1 (c INT) ENGINE=MyISAM; +SET GLOBAL gtid_slave_pos="0-1-100"; +INSERT INTO t1 VALUES (0); +--error ER_BAD_TABLE_ERROR +DROP TABLE not_there; +DROP TABLE t1; + + # Revert table type --disable_query_log --eval ALTER TABLE mysql.gtid_slave_pos ENGINE=$old_slave_pos_engine diff --git a/mysql-test/suite/binlog/t/binlog_base64_flag.test b/mysql-test/suite/binlog/t/binlog_base64_flag.test index 6935f69ba36..75bb8fb4f7c 100644 --- a/mysql-test/suite/binlog/t/binlog_base64_flag.test +++ b/mysql-test/suite/binlog/t/binlog_base64_flag.test @@ -6,7 +6,7 @@ # BINLOG statement does not work in embedded mode. source include/not_embedded.inc; -call mtr.add_suppression("BINLOG_BASE64_EVENT: According to the master's version"); +call mtr.add_suppression("BINLOG_BASE64_EVENT: Master's version is earlier than"); call mtr.add_suppression("BINLOG_BASE64_EVENT: Column 1 of table 'test.char128_utf8' cannot be converted"); CREATE TABLE t1 (a int); diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result index 7131fdd42c2..f6180fcfa28 100644 --- a/mysql-test/suite/encryption/r/tempfiles_encrypted.result +++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result @@ -3545,6 +3545,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables select distinct 1, row_number() over (order by 1) from t1 where a=0 group by a with rollup; 1 row_number() over (order by 1) +1 1 drop table t1; explain SELECT DISTINCT BIT_OR(100) OVER () FROM dual diff --git a/mysql-test/suite/galera/r/galera_fk_char_key.result b/mysql-test/suite/galera/r/galera_fk_char_key.result new file mode 100644 index 00000000000..34f4ac7f9f7 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_fk_char_key.result @@ -0,0 +1,37 @@ +connection node_2; +connection node_1; +connection node_1; +CREATE TABLE p (id CHAR(5) CHARACTER SET utf8mb4 PRIMARY KEY, v INT) ENGINE=InnoDB; +CREATE TABLE c (id INT AUTO_INCREMENT PRIMARY KEY, +p_id CHAR(5) CHARACTER SET utf8mb4 NOT NULL, +KEY k (p_id), +FOREIGN KEY (p_id) REFERENCES p (id)) ENGINE=InnoDB; +INSERT INTO p VALUES (_utf8mb4 0xC3A9, 0); +connection node_2; +SET SESSION wsrep_sync_wait = 1; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_slave_threads = 2; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_1; +INSERT INTO c (p_id) VALUES (_utf8mb4 0xC3A9); +connection node_2; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; +UPDATE p SET v = v + 1 WHERE id = _utf8mb4 0xC3A9; +connection node_2; +SELECT COUNT(*) = 0 AS no_row_modified_in_parallel FROM INFORMATION_SCHEMA.INNODB_TRX WHERE trx_rows_modified > 0; +no_row_modified_in_parallel +1 +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SELECT v FROM p; +v +1 +SELECT COUNT(*) FROM c; +COUNT(*) +1 +SET GLOBAL wsrep_slave_threads = DEFAULT; +SET SESSION wsrep_sync_wait = DEFAULT; +connection node_1; +DROP TABLE c, p; diff --git a/mysql-test/suite/galera/r/galera_fk_char_key_big.result b/mysql-test/suite/galera/r/galera_fk_char_key_big.result new file mode 100644 index 00000000000..d4be5aca312 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_fk_char_key_big.result @@ -0,0 +1,19 @@ +connection node_2; +connection node_1; +connection node_2; +SET GLOBAL wsrep_slave_threads = 2; +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +SET SESSION wsrep_sync_wait = 0; +# Shape 1: CHAR(36) utf8mb4 +# Shape 2: CHAR(36) utf8mb3 +# Shape 3: CHAR(36) utf8mb4, four byte characters +# Shape 4: CHAR(36) utf8mb4_bin +# Shape 5: CHAR(36) utf8mb4, ROW_FORMAT=REDUNDANT +# Shape 6: CHAR(36) utf16 +# Shape 7: CHAR(36) ucs2 +# Shape 8: CHAR(36) latin1 +# Shape 9: CHAR(36) utf8mb4, ASCII value +# Shape 10: VARCHAR(36) utf8mb4 +connection node_2; +SET GLOBAL wsrep_slave_threads = DEFAULT; +disconnect node_2a; diff --git a/mysql-test/suite/galera/r/galera_partitioned_tables.result b/mysql-test/suite/galera/r/galera_partitioned_tables.result index dab39fe0493..b4d9254adc3 100644 --- a/mysql-test/suite/galera/r/galera_partitioned_tables.result +++ b/mysql-test/suite/galera/r/galera_partitioned_tables.result @@ -227,3 +227,21 @@ customer_id customer_name store_id bill_number bill_date amount 8 Oscar S012 125 2019-07-24 300.00 9 Darryl S456 119 2019-08-02 492.20 DROP TABLE sales; +connection node_1; +CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,0),(2,0),(3,0); +CREATE TABLE t2 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB PARTITION BY LIST (a MOD 2) (PARTITION p0 VALUES IN (0), PARTITION p1 VALUES IN (1)) AS SELECT * FROM t1; +SELECT COUNT(*) FROM t2; +COUNT(*) +3 +connection node_2; +connection node_1; +DROP TABLE t1, t2; +CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB; +CREATE TABLE t2 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB PARTITION BY LIST (a MOD 2) (PARTITION p0 VALUES IN (0), PARTITION p1 VALUES IN (1)) AS SELECT * FROM t1; +SELECT COUNT(*) FROM t2; +COUNT(*) +0 +connection node_2; +connection node_1; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/galera/r/galera_toi_alter_auto_increment.result b/mysql-test/suite/galera/r/galera_toi_alter_auto_increment.result index a9923ba4211..ee9f73ebb9f 100644 --- a/mysql-test/suite/galera/r/galera_toi_alter_auto_increment.result +++ b/mysql-test/suite/galera/r/galera_toi_alter_auto_increment.result @@ -20,6 +20,8 @@ MIN(f1) >= 1000 COUNT(*) = 20 COUNT(DISTINCT f1) = 20 1 1 1 connection node_1; ALTER TABLE t1 AUTO_INCREMENT = 5; +Warnings: +Note 4190 ALTER TABLE is implicitly changing the value of 'AUTO_INCREMENT' from '5' to '1039' INSERT INTO t1 (f2) SELECT 1 FROM ten; connection node_2; INSERT INTO t1 (f2) SELECT 1 FROM ten; diff --git a/mysql-test/suite/galera/r/galera_wsrep_new_cluster.result b/mysql-test/suite/galera/r/galera_wsrep_new_cluster.result index 143dee3e6e4..2a85fe3268a 100644 --- a/mysql-test/suite/galera/r/galera_wsrep_new_cluster.result +++ b/mysql-test/suite/galera/r/galera_wsrep_new_cluster.result @@ -1,5 +1,6 @@ connection node_2; connection node_1; +# Correct Galera library found connection node_1; connection node_2; connection node_2; diff --git a/mysql-test/suite/galera/t/galera_fk_char_key.test b/mysql-test/suite/galera/t/galera_fk_char_key.test new file mode 100644 index 00000000000..84a95a9ce2b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_fk_char_key.test @@ -0,0 +1,94 @@ +# +# A foreign key on a CHAR column of a variable length character set. +# +# The key of the child row is built from the MySQL record by +# wsrep_store_key_val_for_row() and the key of the parent row from the InnoDB +# record by wsrep_rec_get_foreign_key(). A CHAR is not space padded the same +# way in the two formats: the MySQL record pads it to n_chars * mbmaxlen +# bytes, while InnoDB strips that padding down to, but not below, n_chars +# bytes. As that compares a byte count with a character count, a value that +# holds a multi byte character and is shorter than the column ended up with a +# different number of characters on the two paths, the two keys differed, the +# child INSERT got no dependency on the parent row and the appliers were free +# to run it in parallel with a change of that very row. +# +# The value used below is one two byte character in a five character column, +# so it is padded, which is what used to make the two encodings diverge. It +# is written as a hex literal so that the test does not depend on the +# character set of the client connection. +# +# The test parks the applier of the child INSERT and replicates an UPDATE of +# the parent row after it, then verifies that the UPDATE had to wait. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/galera_have_debug_sync.inc + +--connection node_1 +CREATE TABLE p (id CHAR(5) CHARACTER SET utf8mb4 PRIMARY KEY, v INT) ENGINE=InnoDB; +CREATE TABLE c (id INT AUTO_INCREMENT PRIMARY KEY, + p_id CHAR(5) CHARACTER SET utf8mb4 NOT NULL, + KEY k (p_id), + FOREIGN KEY (p_id) REFERENCES p (id)) ENGINE=InnoDB; +INSERT INTO p VALUES (_utf8mb4 0xC3A9, 0); + +--connection node_2 +# Catch up with node_1 with a causal read before causal reads are turned off. +SET SESSION wsrep_sync_wait = 1; +--disable_query_log +--disable_result_log +SELECT COUNT(*) FROM p; +--enable_result_log +--enable_query_log +SET SESSION wsrep_sync_wait = 0; + +# Two appliers, so that the two writesets below *can* be applied in parallel +# if certification does not make them depend on each other. +SET GLOBAL wsrep_slave_threads = 2; + +# Park the first applier on entry to the apply monitor, i.e. before the child +# INSERT is applied and before its FK check locks the parent row. +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +INSERT INTO c (p_id) VALUES (_utf8mb4 0xC3A9); + +--connection node_2 +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc + +# Clear the sync point, the second applier must run into the real dependency. +--source include/galera_clear_sync_point.inc + +--let $apply_waits = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_apply_waits'` + +--connection node_1 +UPDATE p SET v = v + 1 WHERE id = _utf8mb4 0xC3A9; + +--connection node_2 +# Wait until the second applier has blocked in the apply monitor because +--let $wait_condition = SELECT (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_apply_waits') > $apply_waits +--source include/wait_condition.inc + +# the second applier waited instead of applying it in parallel. +SELECT COUNT(*) = 0 AS no_row_modified_in_parallel FROM INFORMATION_SCHEMA.INNODB_TRX WHERE trx_rows_modified > 0; + +# Release the child INSERT +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + +--let $wait_condition = SELECT COUNT(*) = 1 FROM c; +--source include/wait_condition.inc +--let $wait_condition = SELECT v = 1 FROM p; +--source include/wait_condition.inc + +SELECT v FROM p; +SELECT COUNT(*) FROM c; + +SET GLOBAL wsrep_slave_threads = DEFAULT; +SET SESSION wsrep_sync_wait = DEFAULT; + +--connection node_1 +DROP TABLE c, p; diff --git a/mysql-test/suite/galera/t/galera_fk_char_key_big.test b/mysql-test/suite/galera/t/galera_fk_char_key_big.test new file mode 100644 index 00000000000..c7da3830eaa --- /dev/null +++ b/mysql-test/suite/galera/t/galera_fk_char_key_big.test @@ -0,0 +1,130 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_sequence.inc + +--connection node_2 +SET GLOBAL wsrep_slave_threads = 2; + +--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +SET SESSION wsrep_sync_wait = 0; + +--let $shape = 0 +while ($shape < 10) +{ + --inc $shape + --let $opt = ENGINE=InnoDB + + if ($shape == 1) + { + --let $name = CHAR(36) utf8mb4 + --let $col = CHAR(36) CHARACTER SET utf8mb4 + --let $val = _utf8mb4 0xC3A9C3A9C3A9C3A9C3A9 + } + if ($shape == 2) + { + --let $name = CHAR(36) utf8mb3 + --let $col = CHAR(36) CHARACTER SET utf8mb3 + --let $val = _utf8mb3 0xC3A9C3A9C3A9C3A9C3A9 + } + if ($shape == 3) + { + --let $name = CHAR(36) utf8mb4, four byte characters + --let $col = CHAR(36) CHARACTER SET utf8mb4 + --let $val = _utf8mb4 0xF09F9880F09F9880F09F9880F09F9880F09F9880 + } + if ($shape == 4) + { + --let $name = CHAR(36) utf8mb4_bin + --let $col = CHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin + --let $val = _utf8mb4 0xC3A9C3A9C3A9C3A9C3A9 + } + if ($shape == 5) + { + --let $name = CHAR(36) utf8mb4, ROW_FORMAT=REDUNDANT + --let $col = CHAR(36) CHARACTER SET utf8mb4 + --let $val = _utf8mb4 0xC3A9C3A9C3A9C3A9C3A9 + --let $opt = ENGINE=InnoDB ROW_FORMAT=REDUNDANT + } + if ($shape == 6) + { + --let $name = CHAR(36) utf16 + --let $col = CHAR(36) CHARACTER SET utf16 + --let $val = _utf16 0x00E900E900E900E900E9 + } + if ($shape == 7) + { + --let $name = CHAR(36) ucs2 + --let $col = CHAR(36) CHARACTER SET ucs2 + --let $val = _ucs2 0x00E900E900E900E900E9 + } + if ($shape == 8) + { + --let $name = CHAR(36) latin1 + --let $col = CHAR(36) CHARACTER SET latin1 + --let $val = _latin1 0xE9E9E9E9E9 + } + if ($shape == 9) + { + --let $name = CHAR(36) utf8mb4, ASCII value + --let $col = CHAR(36) CHARACTER SET utf8mb4 + --let $val = _utf8mb4 0x6162636465 + } + if ($shape == 10) + { + --let $name = VARCHAR(36) utf8mb4 + --let $col = VARCHAR(36) CHARACTER SET utf8mb4 + --let $val = _utf8mb4 0xC3A9C3A9C3A9C3A9C3A9 + } + + --echo # Shape $shape: $name + --disable_connect_log + --connection node_1 + --disable_query_log + --eval CREATE TABLE p (id $col PRIMARY KEY, v INT) $opt + --eval CREATE TABLE c (id INT AUTO_INCREMENT PRIMARY KEY, p_id $col NOT NULL, KEY k (p_id), FOREIGN KEY (p_id) REFERENCES p (id)) $opt + --eval CREATE TABLE f (id INT AUTO_INCREMENT PRIMARY KEY, pad CHAR(255)) $opt + --eval INSERT INTO p VALUES ($val, 0) + --enable_query_log + --enable_connect_log + + --let $i = 0 + --let $round = 10 + while ($round) + { + --inc $i + --disable_connect_log + --connection node_1 + --disable_query_log + BEGIN; + INSERT INTO f (pad) SELECT REPEAT('x',255) FROM seq_1_to_2000; + --eval INSERT INTO c (p_id) VALUES ($val) + COMMIT; + --eval UPDATE p SET v = v + 1 WHERE id = $val + --enable_query_log + + --connection node_2a + --let $wait_timeout = 30 + --let $wait_condition = SELECT COUNT(*) = $i FROM c + --source include/wait_condition.inc + --enable_connect_log + + if (!$success) + { + SELECT ID, STATE, INFO FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'; + SELECT trx_id, trx_state, trx_mysql_thread_id, trx_query FROM INFORMATION_SCHEMA.INNODB_TRX; + --die Node 2 stopped applying. The two appliers are deadlocked on the foreign key parent row. + } + --dec $round + } + + --disable_connect_log + --connection node_1 + --disable_query_log + DROP TABLE c, p, f; + --enable_query_log + --enable_connect_log +} + +--connection node_2 +SET GLOBAL wsrep_slave_threads = DEFAULT; +--disconnect node_2a diff --git a/mysql-test/suite/galera/t/galera_partitioned_tables.test b/mysql-test/suite/galera/t/galera_partitioned_tables.test index 41d2689f171..0cd73e1bf7c 100644 --- a/mysql-test/suite/galera/t/galera_partitioned_tables.test +++ b/mysql-test/suite/galera/t/galera_partitioned_tables.test @@ -167,4 +167,40 @@ SELECT * FROM sales; SET GLOBAL wsrep_mode=DEFAULT; --connection node_2 SELECT * FROM sales; -DROP TABLE sales; \ No newline at end of file +DROP TABLE sales; + +# +# MDEV-40929 : CREATE TABLE ... AS SELECT is not replicated in Galera when the table is partitioned +# + +--connection node_1 +CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,0),(2,0),(3,0); +CREATE TABLE t2 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB PARTITION BY LIST (a MOD 2) (PARTITION p0 VALUES IN (0), PARTITION p1 VALUES IN (1)) AS SELECT * FROM t1; +SELECT COUNT(*) FROM t2; + +--connection node_2 +--let $c= query_get_value(SELECT COUNT(*) AS c FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2', c, 1) +if ($c != 1) { + --die The new partitioned table exists on node 1 but not on node 2. It was not replicated to the cluster. +} +--let $c= query_get_value(SELECT COUNT(*) AS c FROM t2, c, 1) +if ($c != 3) { + --die The new partitioned table on node 2 is missing rows. The rows selected on node 1 were not replicated to the cluster. +} + +--connection node_1 +DROP TABLE t1, t2; + +CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB; +CREATE TABLE t2 (a INT NOT NULL, b INT NOT NULL, PRIMARY KEY (a,b)) ENGINE=InnoDB PARTITION BY LIST (a MOD 2) (PARTITION p0 VALUES IN (0), PARTITION p1 VALUES IN (1)) AS SELECT * FROM t1; +SELECT COUNT(*) FROM t2; + +--connection node_2 +--let $c= query_get_value(SELECT COUNT(*) AS c FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2', c, 1) +if ($c != 1) { + --die The new partitioned table exists on node 1 but not on node 2, with an empty source table. It was not replicated to the cluster. +} + +--connection node_1 +DROP TABLE t1, t2; diff --git a/mysql-test/suite/galera/t/galera_wsrep_new_cluster.test b/mysql-test/suite/galera/t/galera_wsrep_new_cluster.test index 000f91c8bbc..2de79fccd22 100644 --- a/mysql-test/suite/galera/t/galera_wsrep_new_cluster.test +++ b/mysql-test/suite/galera/t/galera_wsrep_new_cluster.test @@ -8,6 +8,10 @@ # This is needed because we use wsrep-new-cluster --source include/have_perfschema.inc +# Make sure that the test is operating on the right version of galera library. +--let $galera_version=26.4.28 +source ../wsrep/include/check_galera_version.inc; + # Save original auto_increment_offset values. --let $node_1=node_1 --let $node_2=node_2 @@ -86,6 +90,11 @@ SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME --echo Starting server ... --let $start_mysqld_params= +# Rejoining with an emptied datadir requires a full SST. That is much slower +# than a plain restart and on a loaded machine it does not finish within the +# default 60 seconds, which used to abort the test here while the SST was +# still running. +--let $galera_wait_ready_timeout=300 --source include/start_mysqld.inc --source include/wait_until_ready.inc diff --git a/mysql-test/suite/innodb/include/assert_blob_not_purged.inc b/mysql-test/suite/innodb/include/assert_blob_not_purged.inc new file mode 100644 index 00000000000..378b6d5031a --- /dev/null +++ b/mysql-test/suite/innodb/include/assert_blob_not_purged.inc @@ -0,0 +1,28 @@ +# Assert that purge has not freed the BLOB while a version-chain walk is parked +# at the dereference of its reference. Purging an update record is what would +# free it; the caller must have enabled and reset that monitor, the counter +# being cumulative over the lifetime of the server. +# +# The wait gives purge every chance to get there first, which is what makes the +# assertion fail rather than pass by accident; without the freeze it needs about +# a second. It cannot be a wait_condition, because a frozen purge_sys.view is +# precisely what stops every purge counter from moving. + +--disable_query_log +let $purge_wait_rounds= 20; +while ($purge_wait_rounds) +{ + let $freed= `SELECT count FROM information_schema.innodb_metrics WHERE name = 'purge_upd_exist_or_extern_records'`; + if ($freed) + { + let $purge_wait_rounds= 0; + } + if (!$freed) + { + real_sleep 0.1; + dec $purge_wait_rounds; + } +} +--enable_query_log +SELECT count AS upd_records_purged_while_parked FROM information_schema.innodb_metrics +WHERE name = 'purge_upd_exist_or_extern_records'; diff --git a/mysql-test/suite/innodb/r/old_blob.result b/mysql-test/suite/innodb/r/old_blob.result new file mode 100644 index 00000000000..3c52fb127f1 --- /dev/null +++ b/mysql-test/suite/innodb/r/old_blob.result @@ -0,0 +1,46 @@ +# +# MDEV-38056 An implicit-lock check must keep purge_sys.view frozen across +# the dereference of an externally stored column of a version it rebuilt. +# The transaction it references commits meanwhile, deregistering itself +# from trx_sys before it waits for the reader's reference, which makes its +# history purgeable under the reader. +# +InnoDB 0 transactions not purged +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; +INSERT INTO t VALUES (1, CONCAT('x', REPEAT('x', @@innodb_page_size))); +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; +BEGIN; +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; +UPDATE t SET b = CONCAT('z', REPEAT('z', @@innodb_page_size)) WHERE a = 1; +connect reader,localhost,root,,; +SET DEBUG_SYNC='row_vers_impl_x_locked_row_build SIGNAL parked WAIT_FOR resume'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE; +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +COMMIT; +connect con3,localhost,root,,; +SELECT count AS upd_records_purged_while_parked FROM information_schema.innodb_metrics +WHERE name = 'purge_upd_exist_or_extern_records'; +upd_records_purged_while_parked +0 +SET DEBUG_SYNC='now SIGNAL resume'; +connection reader; +a +1 +connection default; +disconnect reader; +connection con3; +connection default; +disconnect con3; +SET DEBUG_SYNC='RESET'; +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; diff --git a/mysql-test/suite/innodb/r/old_blob_check.result b/mysql-test/suite/innodb/r/old_blob_check.result new file mode 100644 index 00000000000..ee50220dd6f --- /dev/null +++ b/mysql-test/suite/innodb/r/old_blob_check.result @@ -0,0 +1,57 @@ +# +# MDEV-38056 CHECK TABLE ... EXTENDED must keep purge_sys.view frozen +# across the dereference of an externally stored column of a version it +# rebuilt. It reaches versions that purge_sys.view already permits to be +# freed, deciding reachability from the lagging purge_sys.end_view. +# +InnoDB 0 transactions not purged +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; +INSERT INTO t VALUES (1, CONCAT('z', REPEAT('z', @@innodb_page_size))); +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; +connect purge_control,localhost,root,,; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; +connect victim,localhost,root,,; +BEGIN; +UPDATE t SET b = CONCAT('x', REPEAT('x', @@innodb_page_size)) WHERE a = 1; +connect checker,localhost,root,,; +SET DEBUG_SYNC='row_check_index_extended_match SIGNAL parked WAIT_FOR resume EXECUTE 2'; +CHECK TABLE t EXTENDED; +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +disconnect purge_control; +connection default; +SELECT count AS upd_records_purged_while_parked FROM information_schema.innodb_metrics +WHERE name = 'purge_upd_exist_or_extern_records'; +upd_records_purged_while_parked +0 +SET DEBUG_SYNC='now SIGNAL resume'; +connection checker; +Table Op Msg_type Msg_text +test.t check status OK +disconnect checker; +connection victim; +COMMIT; +connection default; +disconnect victim; +SET DEBUG_SYNC='RESET'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'xx%'; +a +1 +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; diff --git a/mysql-test/suite/innodb/r/old_blob_purgeable.result b/mysql-test/suite/innodb/r/old_blob_purgeable.result new file mode 100644 index 00000000000..c6b665b55b7 --- /dev/null +++ b/mysql-test/suite/innodb/r/old_blob_purgeable.result @@ -0,0 +1,90 @@ +# +# MDEV-38056 Each of the three walks that freeze purge_sys.view across the +# dereference of an externally stored column re-establishes under the +# freeze that the version it rebuilt is still reachable. A debug keyword +# forces each of those tests to fail, so that the exit it guards is +# reachable without the race that produces it. +# +InnoDB 0 transactions not purged +SET @old_dbug= @@SESSION.debug_dbug; +# +# row_vers_impl_x_locked_purgeable: report no implicit lock. +# +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, REPEAT('x', @@innodb_page_size)); +BEGIN; +UPDATE t SET b = REPEAT('y', @@innodb_page_size) WHERE a = 1; +UPDATE t SET b = REPEAT('z', @@innodb_page_size) WHERE a = 1; +connect reader,localhost,root,,; +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,row_vers_impl_x_locked_purgeable'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE SKIP LOCKED; +a +SET SESSION debug_dbug= @old_dbug; +connection default; +COMMIT; +connection reader; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE SKIP LOCKED; +a +1 +COMMIT; +connection default; +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; +# +# row_undo_mod_sec_is_unsafe_purgeable: report the secondary index entry +# as still needed, which only makes the rollback delete-mark an entry it +# could have removed. +# +CREATE TABLE t(a INT PRIMARY KEY, pad INT, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, 0, REPEAT('x', @@innodb_page_size)); +UPDATE t SET pad = 1 WHERE a = 1; +UPDATE t SET b = REPEAT('y', @@innodb_page_size) WHERE a = 1; +BEGIN; +UPDATE t SET b = REPEAT('z', @@innodb_page_size) WHERE a = 1; +SET SESSION debug_dbug='+d,row_undo_mod_sec_is_unsafe_purgeable'; +ROLLBACK; +SET SESSION debug_dbug= @old_dbug; +SELECT a, pad, LEFT(b, 2) FROM t FORCE INDEX(b) WHERE b LIKE 'yy%'; +a pad LEFT(b, 2) +1 1 yy +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; +# +# row_check_index_purgeable: stop the walk and report nothing. +# +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, REPEAT('x', @@innodb_page_size)); +connect purge_control,localhost,root,,; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +UPDATE t SET b = 's' WHERE a = 1; +connect victim,localhost,root,,; +BEGIN; +UPDATE t SET b = 'q' WHERE a = 1; +connection default; +SET SESSION debug_dbug='+d,row_check_index_purgeable'; +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +SET SESSION debug_dbug= @old_dbug; +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +connection victim; +ROLLBACK; +disconnect purge_control; +connection default; +disconnect reader; +disconnect victim; +CHECK TABLE t EXTENDED; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; diff --git a/mysql-test/suite/innodb/r/old_blob_rollback.result b/mysql-test/suite/innodb/r/old_blob_rollback.result new file mode 100644 index 00000000000..0b4408bd84e --- /dev/null +++ b/mysql-test/suite/innodb/r/old_blob_rollback.result @@ -0,0 +1,54 @@ +# +# MDEV-38056 A rollback must keep purge_sys.view frozen across the +# dereference of an externally stored column of a version it rebuilt. It +# froze that view only inside trx_undo_prev_version_build(), so purge was +# free to release the BLOB pages before the caller fetched them. +# +InnoDB 0 transactions not purged +CREATE TABLE t(a INT PRIMARY KEY, pad INT, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; +INSERT INTO t VALUES (1, 0, CONCAT('x', REPEAT('x', @@innodb_page_size))); +INSERT INTO t +SELECT 100 + seq, 0, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; +connect purge_control,localhost,root,,; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +UPDATE t SET pad = 1 WHERE a = 1; +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; +connect victim,localhost,root,,; +BEGIN; +UPDATE t SET b = CONCAT('z', REPEAT('z', @@innodb_page_size)) WHERE a = 1; +SET DEBUG_SYNC='row_undo_mod_sec_is_unsafe_row_build SIGNAL parked WAIT_FOR resume EXECUTE 3'; +ROLLBACK; +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +disconnect purge_control; +connection default; +SELECT count AS upd_records_purged_while_parked FROM information_schema.innodb_metrics +WHERE name = 'purge_upd_exist_or_extern_records'; +upd_records_purged_while_parked +0 +SET DEBUG_SYNC='now SIGNAL resume'; +connection victim; +connection default; +disconnect victim; +SET DEBUG_SYNC='RESET'; +SELECT a, pad FROM t FORCE INDEX(b) WHERE b LIKE 'yy%'; +a pad +1 1 +CHECK TABLE t; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; diff --git a/mysql-test/suite/innodb/r/old_blob_updel.result b/mysql-test/suite/innodb/r/old_blob_updel.result new file mode 100644 index 00000000000..68e252c8695 --- /dev/null +++ b/mysql-test/suite/innodb/r/old_blob_updel.result @@ -0,0 +1,72 @@ +# +# MDEV-38056 As innodb.old_blob, but the version is rebuilt from the undo +# log record of an update of a record that another, committed transaction +# delete-marked. Such a record stores only the 20-byte reference, so the +# frozen purge_sys.view is the only thing that can keep the dereference +# valid. Reaching that version needs a batch to be still in progress. +# +InnoDB 0 transactions not purged +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; +INSERT INTO t VALUES (1, CONCAT('x', REPEAT('x', @@innodb_page_size))); +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; +SET GLOBAL innodb_monitor_reset_all='purge_del_mark_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_del_mark_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; +connect purge_control,localhost,root,,; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +DELETE FROM t WHERE a = 1; +connect victim,localhost,root,,; +BEGIN; +INSERT INTO t VALUES (1, CONCAT('z', REPEAT('z', @@innodb_page_size))); +connection default; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug='+d,purge_hold_cleanup'; +disconnect purge_control; +connection default; +SET DEBUG_SYNC='now WAIT_FOR purge_batch_parked TIMEOUT 60'; +SELECT count > 0 AS del_mark_records_purged +FROM information_schema.innodb_metrics WHERE name = 'purge_del_mark_records'; +del_mark_records_purged +1 +connect reader,localhost,root,,; +SET DEBUG_SYNC='row_vers_impl_x_locked_row_build SIGNAL parked WAIT_FOR resume'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE; +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET GLOBAL debug_dbug= @old_dbug; +SET DEBUG_SYNC='now SIGNAL purge_batch_resume'; +connection victim; +COMMIT; +connection default; +SELECT count AS upd_records_purged_while_parked FROM information_schema.innodb_metrics +WHERE name = 'purge_upd_exist_or_extern_records'; +upd_records_purged_while_parked +0 +SET DEBUG_SYNC='now SIGNAL resume'; +connection victim; +connection reader; +a +1 +connection default; +disconnect reader; +disconnect victim; +SET DEBUG_SYNC='RESET'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'zz%'; +a +1 +CHECK TABLE t; +Table Op Msg_type Msg_text +test.t check status OK +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_del_mark_records'; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_del_mark_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; diff --git a/mysql-test/suite/innodb/t/old_blob.test b/mysql-test/suite/innodb/t/old_blob.test new file mode 100644 index 00000000000..85799345a22 --- /dev/null +++ b/mysql-test/suite/innodb/t/old_blob.test @@ -0,0 +1,75 @@ +--source include/have_innodb.inc +--source include/have_innodb_max_16k.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +--echo # +--echo # MDEV-38056 An implicit-lock check must keep purge_sys.view frozen across +--echo # the dereference of an externally stored column of a version it rebuilt. +--echo # The transaction it references commits meanwhile, deregistering itself +--echo # from trx_sys before it waits for the reader's reference, which makes its +--echo # history purgeable under the reader. +--echo # + +--source include/wait_all_purged.inc + +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; + +INSERT INTO t VALUES (1, CONCAT('x', REPEAT('x', @@innodb_page_size))); + +# Filler rows keep the delete-marked entries off the leaf page holding the live +# 'z...' entry, which the reader latches; purge would block on it otherwise. +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; + +# The counter is cumulative over the lifetime of the server. +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; + +BEGIN; +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; +UPDATE t SET b = CONCAT('z', REPEAT('z', @@innodb_page_size)) WHERE a = 1; + +# The locking read references the still active transaction above and parks after +# building the previous version, before dereferencing its BLOB pointers. +connect (reader,localhost,root,,); +SET DEBUG_SYNC='row_vers_impl_x_locked_row_build SIGNAL parked WAIT_FOR resume'; +--send SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE + +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +--send COMMIT + +# Both old BLOBs must still be allocated while the reader is parked on them. +connect (con3,localhost,root,,); +--source suite/innodb/include/assert_blob_not_purged.inc +SET DEBUG_SYNC='now SIGNAL resume'; + +connection reader; +--reap + +connection default; +--reap +disconnect reader; + +# Once the reader is done, purge advances, which confirms that only the frozen +# view held it back. +connection con3; +let $wait_condition= + SELECT count >= 2 FROM information_schema.innodb_metrics + WHERE name = 'purge_upd_exist_or_extern_records'; +--source include/wait_condition.inc + +connection default; +disconnect con3; +SET DEBUG_SYNC='RESET'; +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +--disable_warnings +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; +--enable_warnings diff --git a/mysql-test/suite/innodb/t/old_blob_check.test b/mysql-test/suite/innodb/t/old_blob_check.test new file mode 100644 index 00000000000..bde15c5e50d --- /dev/null +++ b/mysql-test/suite/innodb/t/old_blob_check.test @@ -0,0 +1,96 @@ +--source include/have_innodb.inc +--source include/have_innodb_max_16k.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +--echo # +--echo # MDEV-38056 CHECK TABLE ... EXTENDED must keep purge_sys.view frozen +--echo # across the dereference of an externally stored column of a version it +--echo # rebuilt. It reaches versions that purge_sys.view already permits to be +--echo # freed, deciding reachability from the lagging purge_sys.end_view. +--echo # + +--source include/wait_all_purged.inc + +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; + +INSERT INTO t VALUES (1, CONCAT('z', REPEAT('z', @@innodb_page_size))); + +# Filler rows keep the delete-marked 'z...' entry off the first leaf page, which +# CHECK TABLE holds while parked; purge would block on it otherwise. +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; + +# The counter is cumulative over the lifetime of the server. +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; + +# Pin a read view on the 'z...' state, so that the BLOB the update below disowns +# stays allocated until we release it. +connect (purge_control,localhost,root,,); +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +connection default; +# Committed update that disowns the 'z...' BLOB, which purge will free. +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; + +# An active transaction keeps the newest version invisible to CHECK TABLE, which +# is what makes it descend the version chain at all. +connect (victim,localhost,root,,); +BEGIN; +UPDATE t SET b = CONCAT('x', REPEAT('x', @@innodb_page_size)) WHERE a = 1; + +# While checking the live 'x...' entry, CHECK TABLE rebuilds the 'y...' and then +# the 'z...' version and compares the column prefix of each against it. +connect (checker,localhost,root,,); +SET DEBUG_SYNC='row_check_index_extended_match SIGNAL parked WAIT_FOR resume EXECUTE 2'; +--send CHECK TABLE t EXTENDED + +connection default; +# Hit 1: the 'y...' version, whose BLOB the active transaction above disowned and +# which is therefore not freeable. +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; + +# Hit 2: the 'z...' version, whose BLOB the committed update disowned. Release +# the read view, the last thing that kept that BLOB from being freeable. Purge +# has to remove the delete-marked 'z...' entry first, and that sits on the last +# leaf page while CHECK TABLE holds the first. +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +disconnect purge_control; + +connection default; +--source suite/innodb/include/assert_blob_not_purged.inc +SET DEBUG_SYNC='now SIGNAL resume'; + +connection checker; +--reap +disconnect checker; + +connection victim; +COMMIT; + +connection default; +disconnect victim; + +# Once CHECK TABLE is done, purge advances and frees that BLOB, which confirms +# that only the frozen view held it back. +let $wait_condition= + SELECT count >= 1 FROM information_schema.innodb_metrics + WHERE name = 'purge_upd_exist_or_extern_records'; +--source include/wait_condition.inc + +SET DEBUG_SYNC='RESET'; + +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'xx%'; +CHECK TABLE t EXTENDED; +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +--disable_warnings +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; +--enable_warnings diff --git a/mysql-test/suite/innodb/t/old_blob_purgeable.test b/mysql-test/suite/innodb/t/old_blob_purgeable.test new file mode 100644 index 00000000000..aef0c365585 --- /dev/null +++ b/mysql-test/suite/innodb/t/old_blob_purgeable.test @@ -0,0 +1,111 @@ +--source include/have_innodb.inc +--source include/have_debug.inc + +--echo # +--echo # MDEV-38056 Each of the three walks that freeze purge_sys.view across the +--echo # dereference of an externally stored column re-establishes under the +--echo # freeze that the version it rebuilt is still reachable. A debug keyword +--echo # forces each of those tests to fail, so that the exit it guards is +--echo # reachable without the race that produces it. +--echo # + +--source include/wait_all_purged.inc + +SET @old_dbug= @@SESSION.debug_dbug; + +--echo # +--echo # row_vers_impl_x_locked_purgeable: report no implicit lock. +--echo # + +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, REPEAT('x', @@innodb_page_size)); + +BEGIN; +UPDATE t SET b = REPEAT('y', @@innodb_page_size) WHERE a = 1; +UPDATE t SET b = REPEAT('z', @@innodb_page_size) WHERE a = 1; + +# The implicit lock check walks the versions of the row this transaction still +# holds, and row_vers_impl_x_locked_purgeable makes it answer that the row is +# unlocked. The reader skips the row anyway, the clustered index record carrying +# the same implicit lock, so the row stays protected. +connect (reader,localhost,root,,); +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,row_vers_impl_x_locked_purgeable'; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE SKIP LOCKED; +SET SESSION debug_dbug= @old_dbug; + +connection default; +COMMIT; + +connection reader; +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE SKIP LOCKED; +COMMIT; + +connection default; +CHECK TABLE t EXTENDED; +DROP TABLE t; + +--echo # +--echo # row_undo_mod_sec_is_unsafe_purgeable: report the secondary index entry +--echo # as still needed, which only makes the rollback delete-mark an entry it +--echo # could have removed. +--echo # + +CREATE TABLE t(a INT PRIMARY KEY, pad INT, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, 0, REPEAT('x', @@innodb_page_size)); +# The pad update produces a version that merely inherits the reference to the +# 'x...' BLOB, which is what the walk below has to dereference. +UPDATE t SET pad = 1 WHERE a = 1; +UPDATE t SET b = REPEAT('y', @@innodb_page_size) WHERE a = 1; + +BEGIN; +UPDATE t SET b = REPEAT('z', @@innodb_page_size) WHERE a = 1; +SET SESSION debug_dbug='+d,row_undo_mod_sec_is_unsafe_purgeable'; +ROLLBACK; +SET SESSION debug_dbug= @old_dbug; + +SELECT a, pad, LEFT(b, 2) FROM t FORCE INDEX(b) WHERE b LIKE 'yy%'; +CHECK TABLE t EXTENDED; +DROP TABLE t; + +--echo # +--echo # row_check_index_purgeable: stop the walk and report nothing. +--echo # + +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC; +INSERT INTO t VALUES (1, REPEAT('x', @@innodb_page_size)); + +# Keep the 'x...' version reachable, so that the walk gets to dereference it. +connect (purge_control,localhost,root,,); +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +connection default; +# Two short values, so that only the third version down stores its column +# externally, and the walk reaches the freeze only there. Stopping that late +# costs no count: a version that the check can see was rejected higher up, so +# the record was never going to be counted, and the check still says OK. +UPDATE t SET b = 's' WHERE a = 1; + +connect (victim,localhost,root,,); +BEGIN; +UPDATE t SET b = 'q' WHERE a = 1; + +connection default; +SET SESSION debug_dbug='+d,row_check_index_purgeable'; +CHECK TABLE t EXTENDED; +SET SESSION debug_dbug= @old_dbug; +CHECK TABLE t EXTENDED; + +connection victim; +ROLLBACK; + +disconnect purge_control; + +connection default; +disconnect reader; +disconnect victim; +CHECK TABLE t EXTENDED; +DROP TABLE t; diff --git a/mysql-test/suite/innodb/t/old_blob_rollback.test b/mysql-test/suite/innodb/t/old_blob_rollback.test new file mode 100644 index 00000000000..681e7789d52 --- /dev/null +++ b/mysql-test/suite/innodb/t/old_blob_rollback.test @@ -0,0 +1,99 @@ +--source include/have_innodb.inc +--source include/have_innodb_max_16k.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +--echo # +--echo # MDEV-38056 A rollback must keep purge_sys.view frozen across the +--echo # dereference of an externally stored column of a version it rebuilt. It +--echo # froze that view only inside trx_undo_prev_version_build(), so purge was +--echo # free to release the BLOB pages before the caller fetched them. +--echo # + +--source include/wait_all_purged.inc + +CREATE TABLE t(a INT PRIMARY KEY, pad INT, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; + +INSERT INTO t VALUES (1, 0, CONCAT('x', REPEAT('x', @@innodb_page_size))); + +# Filler rows keep the delete-marked entries off the leaf page holding the live +# 'z...' entry, which the rollback latches; purge would block on it otherwise. +INSERT INTO t +SELECT 100 + seq, 0, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; + +# The counter is cumulative over the lifetime of the server. +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; + +# Pin a read view on the 'x...' state, so that the BLOB the updates below disown +# stays allocated until we release it. +connect (purge_control,localhost,root,,); +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +connection default; +# Committed update that does not mention b: the version it produces inherits +# whatever reference b holds in the next newer version. +UPDATE t SET pad = 1 WHERE a = 1; +# Committed update that disowns the 'x...' BLOB, which purge will free. +UPDATE t SET b = CONCAT('y', REPEAT('y', @@innodb_page_size)) WHERE a = 1; + +connect (victim,localhost,root,,); +BEGIN; +UPDATE t SET b = CONCAT('z', REPEAT('z', @@innodb_page_size)) WHERE a = 1; + +# On ROLLBACK the walk is 'z...' -> 'y...' -> 'x...' -> 'x...' inherited. Park +# before each of the three row_build() calls. +SET DEBUG_SYNC='row_undo_mod_sec_is_unsafe_row_build SIGNAL parked WAIT_FOR resume EXECUTE 3'; +--send ROLLBACK + +connection default; +# Hit 1: the 'y...' version, whose BLOB belongs to the rolling back +# transaction and is never freeable. +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; + +# Hit 2: the 'x...' version, rebuilt from the b update's undo log record. The +# read view still pins that BLOB, so there is nothing to observe here. +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +SET DEBUG_SYNC='now SIGNAL resume'; + +# Hit 3: the version the pad update produced. Its undo log record says nothing +# about b, so it merely inherits the reference that the b update disowned, and a +# frozen purge_sys.view is the only thing that can keep the dereference valid. +# Release the read view, the last thing that kept that BLOB from being freeable. +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +disconnect purge_control; + +connection default; +--source suite/innodb/include/assert_blob_not_purged.inc +SET DEBUG_SYNC='now SIGNAL resume'; + +connection victim; +--reap + +connection default; +disconnect victim; + +# Once the rollback is over, purge advances past both committed updates, which +# confirms that only the frozen view held it back. Only the b update frees a +# BLOB, the pad update having disowned none. +let $wait_condition= + SELECT count >= 2 FROM information_schema.innodb_metrics + WHERE name = 'purge_upd_exist_or_extern_records'; +--source include/wait_condition.inc + +SET DEBUG_SYNC='RESET'; + +# The rollback reverted to the committed 'y...' value, and the indexes agree. +SELECT a, pad FROM t FORCE INDEX(b) WHERE b LIKE 'yy%'; +CHECK TABLE t; +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +--disable_warnings +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; +--enable_warnings diff --git a/mysql-test/suite/innodb/t/old_blob_updel.test b/mysql-test/suite/innodb/t/old_blob_updel.test new file mode 100644 index 00000000000..4020466184f --- /dev/null +++ b/mysql-test/suite/innodb/t/old_blob_updel.test @@ -0,0 +1,118 @@ +--source include/have_innodb.inc +--source include/have_innodb_max_16k.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +--echo # +--echo # MDEV-38056 As innodb.old_blob, but the version is rebuilt from the undo +--echo # log record of an update of a record that another, committed transaction +--echo # delete-marked. Such a record stores only the 20-byte reference, so the +--echo # frozen purge_sys.view is the only thing that can keep the dereference +--echo # valid. Reaching that version needs a batch to be still in progress. +--echo # + +--source include/wait_all_purged.inc + +CREATE TABLE t(a INT PRIMARY KEY, b TEXT, UNIQUE(b(700))) +ENGINE=InnoDB STATS_PERSISTENT=0 ROW_FORMAT=COMPRESSED; + +INSERT INTO t VALUES (1, CONCAT('x', REPEAT('x', @@innodb_page_size))); + +# Filler rows keep the delete-marked 'x...' entry off the leaf page holding the +# live 'z...' entry, which the reader latches; purge would block on it otherwise. +INSERT INTO t +SELECT 100 + seq, CONCAT('y~', LPAD(seq, 3, '0'), REPEAT('-', 800)) +FROM seq_60_to_1; + +# The counters are cumulative over the lifetime of the server. +SET GLOBAL innodb_monitor_reset_all='purge_del_mark_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_enable='purge_del_mark_records'; +SET GLOBAL innodb_monitor_enable='purge_upd_exist_or_extern_records'; + +# Pin purge so that the DELETE below stays unpurged until we choose. +connect (purge_control,localhost,root,,); +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +connection default; +# T1 delete-marks the row; the 'x...' BLOB is still owned by the record. +DELETE FROM t WHERE a = 1; + +# T2 re-inserts the same PRIMARY KEY: an update of a record that T1 +# delete-marked, so the undo log record stores only the reference, no prefix. +connect (victim,localhost,root,,); +BEGIN; +INSERT INTO t VALUES (1, CONCAT('z', REPEAT('z', @@innodb_page_size))); + +# Hold the batch below open once it has purged T1, which is the window in which +# purge_sys.end_view still presents that history as reconstructible. +connection default; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug='+d,purge_hold_cleanup'; + +# Let purge see T1 and remove the delete-marked 'x...' entry: nobody holds the +# clustered index leaf yet. T2 is still active, so purge stops there. +disconnect purge_control; + +connection default; +SET DEBUG_SYNC='now WAIT_FOR purge_batch_parked TIMEOUT 60'; + +# The batch is parked, so its last purged record is behind it and the counter is +# final. It removed the delete-marked 'x...' entry. +SELECT count > 0 AS del_mark_records_purged +FROM information_schema.innodb_metrics WHERE name = 'purge_del_mark_records'; + +# The reader references the active T2. end_view cannot see T1, so it rebuilds the +# delete-marked 'x...' version and parks before dereferencing it. +connect (reader,localhost,root,,); +SET DEBUG_SYNC='row_vers_impl_x_locked_row_build SIGNAL parked WAIT_FOR resume'; +--send SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'z%' FOR UPDATE + +connection default; +SET DEBUG_SYNC='now WAIT_FOR parked TIMEOUT 60'; +# The reader has descended, so the batch may finish. The next one cannot start +# before the reader is done anyway, the reader holding the freeze. Clear +# purge_hold_cleanup first, so that the next batch does not park. +SET GLOBAL debug_dbug= @old_dbug; +SET DEBUG_SYNC='now SIGNAL purge_batch_resume'; + +# T2 commits, making purgeable the undo log record that disowned the 'x...' BLOB, +# and then waits for the reference the parked reader holds. +connection victim; +--send COMMIT + +connection default; +--source suite/innodb/include/assert_blob_not_purged.inc + +SET DEBUG_SYNC='now SIGNAL resume'; + +connection victim; +--reap +connection reader; +--reap + +connection default; +disconnect reader; +disconnect victim; +SET DEBUG_SYNC='RESET'; + +# Once the reader is done, purge advances and frees that BLOB, which confirms +# that only the frozen view held it back. +let $wait_condition= + SELECT count >= 1 FROM information_schema.innodb_metrics + WHERE name = 'purge_upd_exist_or_extern_records'; +--source include/wait_condition.inc + +SELECT a FROM t FORCE INDEX(b) WHERE b LIKE 'zz%'; +CHECK TABLE t; +DROP TABLE t; +SET GLOBAL innodb_monitor_disable='purge_del_mark_records'; +SET GLOBAL innodb_monitor_disable='purge_upd_exist_or_extern_records'; +SET GLOBAL innodb_monitor_reset_all='purge_del_mark_records'; +SET GLOBAL innodb_monitor_reset_all='purge_upd_exist_or_extern_records'; +--disable_warnings +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; +--enable_warnings diff --git a/mysql-test/suite/innodb_fts/r/create.result b/mysql-test/suite/innodb_fts/r/create.result index 04fc20195b6..1147b26cfd2 100644 --- a/mysql-test/suite/innodb_fts/r/create.result +++ b/mysql-test/suite/innodb_fts/r/create.result @@ -211,3 +211,11 @@ ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Not implemented for sys ALTER TABLE t1 ADD FULLTEXT INDEX(b), ALGORITHM=COPY; ERROR HY000: Index 'FTS_DOC_ID_INDEX' is of wrong type for an InnoDB FULLTEXT index DROP TABLE t1; +# +# MDEV-40985 row_end access out of bounds in ALTER TABLE +# +CREATE TABLE t1 +(a TEXT,b INT AS(1),c INT AS(1),d INT AS(1),e INT AS(1),f INT AS(1), +FULLTEXT INDEX(a)) ENGINE=InnoDB; +ALTER TABLE t1 FORCE; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/r/fulltext.result b/mysql-test/suite/innodb_fts/r/fulltext.result index 39d8f49de8d..8a2a786c822 100644 --- a/mysql-test/suite/innodb_fts/r/fulltext.result +++ b/mysql-test/suite/innodb_fts/r/fulltext.result @@ -844,5 +844,18 @@ id select_type table type possible_keys key key_len ref rows Extra 2 SUBQUERY t2 fulltext b b 0 1 Using where DROP TABLE t1, t2; # +# MDEV-41022 Wildcard term returns no rows when combined with a +# phrase in boolean search +# +CREATE TABLE t1(f1 TEXT, FULLTEXT(f1)) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('auth code out'); +SELECT * FROM t1 WHERE MATCH(f1) AGAINST('"zzzz" aut*' IN BOOLEAN MODE); +f1 +auth code out +SELECT * FROM t1 WHERE MATCH(f1) AGAINST('"zzzz yyyy" @3 aut*' IN BOOLEAN MODE); +f1 +auth code out +DROP TABLE t1; +# # End of 10.11 tests # diff --git a/mysql-test/suite/innodb_fts/t/create.test b/mysql-test/suite/innodb_fts/t/create.test index 9d88c1eddd4..e4f21515077 100644 --- a/mysql-test/suite/innodb_fts/t/create.test +++ b/mysql-test/suite/innodb_fts/t/create.test @@ -142,3 +142,13 @@ ALTER TABLE t1 ADD FULLTEXT INDEX(b), ALGORITHM=INPLACE; --error ER_INNODB_FT_WRONG_DOCID_INDEX ALTER TABLE t1 ADD FULLTEXT INDEX(b), ALGORITHM=COPY; DROP TABLE t1; + +--echo # +--echo # MDEV-40985 row_end access out of bounds in ALTER TABLE +--echo # + +CREATE TABLE t1 +(a TEXT,b INT AS(1),c INT AS(1),d INT AS(1),e INT AS(1),f INT AS(1), + FULLTEXT INDEX(a)) ENGINE=InnoDB; +ALTER TABLE t1 FORCE; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/fulltext.test b/mysql-test/suite/innodb_fts/t/fulltext.test index 6318f16883c..144c6611c2d 100644 --- a/mysql-test/suite/innodb_fts/t/fulltext.test +++ b/mysql-test/suite/innodb_fts/t/fulltext.test @@ -851,6 +851,16 @@ WHERE EXISTS ( # Cleanup DROP TABLE t1, t2; +--echo # +--echo # MDEV-41022 Wildcard term returns no rows when combined with a +--echo # phrase in boolean search +--echo # +CREATE TABLE t1(f1 TEXT, FULLTEXT(f1)) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('auth code out'); +SELECT * FROM t1 WHERE MATCH(f1) AGAINST('"zzzz" aut*' IN BOOLEAN MODE); +SELECT * FROM t1 WHERE MATCH(f1) AGAINST('"zzzz yyyy" @3 aut*' IN BOOLEAN MODE); +DROP TABLE t1; + --echo # --echo # End of 10.11 tests --echo # diff --git a/mysql-test/suite/sys_vars/r/gtid_slave_pos_basic.result b/mysql-test/suite/sys_vars/r/gtid_slave_pos_basic.result index 3d880418f31..5402838ed9f 100644 --- a/mysql-test/suite/sys_vars/r/gtid_slave_pos_basic.result +++ b/mysql-test/suite/sys_vars/r/gtid_slave_pos_basic.result @@ -28,6 +28,20 @@ SET GLOBAL gtid_slave_pos= '1-2-3 '; ERROR HY000: Could not parse GTID list SET GLOBAL gtid_slave_pos= '1-2-3,2-4'; ERROR HY000: Could not parse GTID list +SET GLOBAL gtid_slave_pos= '0-1-10,1-1-20,2-1-30'; +SET GLOBAL gtid_slave_pos= '0-1-110,1-1-120,,2-1-130'; +ERROR HY000: Could not parse GTID list +SELECT @@GLOBAL.gtid_slave_pos; +@@GLOBAL.gtid_slave_pos +0-1-10,1-1-20,2-1-30 +SELECT A.domain_id, A.server_id, A.seq_no +FROM mysql.gtid_slave_pos A +WHERE A.sub_id=(SELECT MAX(B.sub_id) FROM mysql.gtid_slave_pos B WHERE A.domain_id=B.domain_id) +ORDER BY A.domain_id; +domain_id server_id seq_no +0 1 10 +1 1 20 +2 1 30 SET GLOBAL gtid_slave_pos= '0-1-10,0-2-20'; ERROR HY000: GTID 0-2-20 and 0-1-10 conflict (duplicate domain id 0) SET GLOBAL gtid_slave_pos= '0-1-10,1-2-20,2-3-30,1-20-200,3-4-1'; diff --git a/mysql-test/suite/sys_vars/t/gtid_slave_pos_basic.test b/mysql-test/suite/sys_vars/t/gtid_slave_pos_basic.test index d021e335b86..b300c81f3e0 100644 --- a/mysql-test/suite/sys_vars/t/gtid_slave_pos_basic.test +++ b/mysql-test/suite/sys_vars/t/gtid_slave_pos_basic.test @@ -24,6 +24,17 @@ SET GLOBAL gtid_slave_pos= '1-2-3 '; --error ER_INCORRECT_GTID_STATE SET GLOBAL gtid_slave_pos= '1-2-3,2-4'; +# Test that a parse error in the middle of the GTID string does not leave +# a partial updated value. +SET GLOBAL gtid_slave_pos= '0-1-10,1-1-20,2-1-30'; +--error ER_INCORRECT_GTID_STATE +SET GLOBAL gtid_slave_pos= '0-1-110,1-1-120,,2-1-130'; +SELECT @@GLOBAL.gtid_slave_pos; +SELECT A.domain_id, A.server_id, A.seq_no + FROM mysql.gtid_slave_pos A + WHERE A.sub_id=(SELECT MAX(B.sub_id) FROM mysql.gtid_slave_pos B WHERE A.domain_id=B.domain_id) + ORDER BY A.domain_id; + --error ER_DUPLICATE_GTID_DOMAIN SET GLOBAL gtid_slave_pos= '0-1-10,0-2-20'; --error ER_DUPLICATE_GTID_DOMAIN diff --git a/mysql-test/suite/wsrep/r/wsrep_protocol_versions.result b/mysql-test/suite/wsrep/r/wsrep_protocol_versions.result index 0d2759aaa89..f81ab8cef1a 100644 --- a/mysql-test/suite/wsrep/r/wsrep_protocol_versions.result +++ b/mysql-test/suite/wsrep/r/wsrep_protocol_versions.result @@ -1,7 +1,7 @@ # Correct Galera library found show status like 'wsrep_protocol%'; Variable_name Value -wsrep_protocol_application 4 +wsrep_protocol_application 5 wsrep_protocol_gcs 6 wsrep_protocol_replicator 11 wsrep_protocol_version 11 diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 73cd5251871..e6896baaad0 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -234,6 +234,19 @@ get_socat_ver() fi } +check_for_socat_commonname() +{ + [ -n "${SOCAT_COMMONNAME+x}" ] && return + # Some socat builds don't register "commonname" as an option at all + # (parseopts() rejects it as unknown regardless of value), so probe + # the actual binary instead of relying on the socat version number. + if socat -hhh 2>&1 | grep -qw 'openssl-commonname'; then + SOCAT_COMMONNAME=1 + else + SOCAT_COMMONNAME=0 + fi +} + get_transfer() { if [ "$tfmt" = 'nc' ]; then @@ -434,6 +447,15 @@ get_transfer() exit 22 fi + if [ -n "$CN_option" ]; then + check_for_socat_commonname + if [ "$SOCAT_COMMONNAME" -eq 0 ]; then + wsrep_log_info \ + "socat lacks the 'commonname' option; skipping peer name verification" + CN_option="" + fi + fi + tcmd="$tcmd$CN_option$sockopt" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then diff --git a/sql/field.cc b/sql/field.cc index ff944547663..f42cba02bd3 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7724,7 +7724,7 @@ Field_string::compatible_field_size(uint field_metadata, { #ifdef HAVE_REPLICATION const Check_field_param check_param = { this }; - if (rpl_master_has_bug(rli_arg, 37426, TRUE, + if (rpl_master_has_bug(rli_arg, RPL_BUG_MYSQL_37426, TRUE, check_field_for_37426, &check_param)) return FALSE; // Not compatible field sizes #endif diff --git a/sql/log_event.h b/sql/log_event.h index 7c40f5db717..60321e3dabe 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -162,6 +162,26 @@ class String; #define NUM_LOAD_DELIM_STRS 5 + +/* + Master-side bugs that we work-around with rpl_master_has_bug(). + Each define is a bit number (0, 1, 2, ...) set in + Relay_log_info::rpl_master_bug_bitmask. + These bit numbers are internal-only, can be changed (eg. to drop a bug + work-around for ancient version) without concern for backwards compatibility. +*/ +enum enum_rpl_master_bug_ids { + RPL_BUG_MYSQL_24432, + RPL_BUG_MYSQL_33029, + RPL_BUG_MYSQL_37426, + RPL_BUG_MDEV_29621, + RPL_BUG_END /* To check size of Relay_log_info::rpl_master_bug_bitmask */ +}; + +/* Must be large enough to hold all enum_rpl_master_bug_ids bits. */ +typedef uint32_t rpl_bug_id_t; + + /* The following is the max table_map_id. This is limited by that we are using 6 bytes for it in replication diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 86068230117..22d076823a4 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -2578,6 +2578,14 @@ int Format_description_log_event::do_apply_event(rpl_group_info *rgi) copy_crypto_data(rli->relay_log.description_event_for_exec); delete rli->relay_log.description_event_for_exec; rli->relay_log.description_event_for_exec= this; + /* + The start of the relay log has our own generated format description + event followed by the real format description from the master. + Don't re-compute the bug bitmask from the generated event so we don't + leave a window where a wrong value is temporarily set. + */ + if (!is_relay_log_event()) + rli->calc_master_bug_bitmask(this); } DBUG_RETURN(ret); @@ -6985,8 +6993,8 @@ int Rows_log_event::update_sequence() (table_rgi && !(table_rgi->gtid_ev_flags2 & Gtid_log_event::FL_DDL) && !(old_master= - rpl_master_has_bug(thd_rgi->rli, - 29621, FALSE, FALSE, FALSE, TRUE)))) + rpl_master_has_bug(thd_rgi->rli, RPL_BUG_MDEV_29621, FALSE, + NULL, NULL)))) { /* This event come from a setval function executed on the master. Update the sequence next_number and round, like we do with setval() diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index a7eeafcb4f6..7ab0d5be08a 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -7018,7 +7018,7 @@ bool JOIN::choose_tableless_subquery_plan() */ if (zero_result_cause) { - if (!implicit_grouping) + if (!implicit_grouping && select_lex->olap != ROLLUP_TYPE) { /* Both group by queries and non-group by queries without aggregate diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 5dfb97902a4..ad6563c196c 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1404,6 +1404,7 @@ rpl_slave_state::load(THD *thd, const char *state_from_master, size_t len, } if (state_from_master == end) return 0; + int err= 0; for (;;) { rpl_gtid gtid; @@ -1414,14 +1415,26 @@ rpl_slave_state::load(THD *thd, const char *state_from_master, size_t len, !(sub_id= next_sub_id(gtid.domain_id)) || record_gtid(thd, >id, sub_id, false, in_statement, &hton) || update(gtid.domain_id, gtid.server_id, sub_id, gtid.seq_no, hton, NULL)) - return 1; + { + err= 1; + break; + } if (state_from_master == end) break; if (*state_from_master != ',') - return 1; + { + err= 1; + break; + } ++state_from_master; } - return 0; + + if (unlikely(err)) + trans_rollback(thd); + else + err= trans_commit(thd); + + return err; } diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 3954541b510..d0909963546 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -54,7 +54,8 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery, const char* thread_name) cur_log_old_open_count(0), error_on_rli_init_info(false), group_relay_log_pos(0), event_relay_log_pos(0), group_master_log_pos(0), log_space_total(0), ignore_log_space_limit(0), - last_master_timestamp(0), sql_thread_caught_up(true), slave_skip_counter(0), + last_master_timestamp(0), sql_thread_caught_up(true), + rpl_master_bug_bitmask(0), slave_skip_counter(0), abort_pos_wait(0), slave_run_id(0), sql_driver_thd(), gtid_skip_flag(GTID_SKIP_NOT), inited(0), abort_slave(0), stop_for_until(0), slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), @@ -729,6 +730,12 @@ int init_relay_log_pos(Relay_log_info* rli,const char* log, goto err; delete rli->relay_log.description_event_for_exec; rli->relay_log.description_event_for_exec= fdev; + /* + The read_relay_log_description_event() returns the real FD from the + master found near the start of the relay log. Initialize the bitmask + of any known bugs in that master version. + */ + rli->calc_master_bug_bitmask(fdev); } my_b_seek(rli->cur_log,(off_t)pos); DBUG_PRINT("info", ("my_b_tell(rli->cur_log)=%llu rli->event_relay_log_pos=%llu", @@ -2645,4 +2652,159 @@ bool Relay_log_info::flush() DBUG_RETURN(error); } + +struct st_version_range_for_one_bug { + rpl_bug_id_t bug_id; + uint bug_num; + bool mysql_bug; // Else Mariadb MDEV + Version introduced_in; // first version with bug + Version fixed_in; // first version with fix +}; +static struct st_version_range_for_one_bug versions_for_all_bugs[]= +{ + /* + Sorted with the most recent server version first, since this is the version + that will be printed in any error messages. + */ + /* MySQL bugs. */ + {RPL_BUG_MYSQL_24432, 24432, true, { 5, 1, 12 }, { 5, 1, 17 } }, + {RPL_BUG_MYSQL_24432, 24432, true, { 5, 0, 24 }, { 5, 0, 38 } }, + {RPL_BUG_MYSQL_33029, 33029, true, { 5, 1, 0 }, { 5, 1, 12 } }, + {RPL_BUG_MYSQL_33029, 33029, true, { 5, 0, 0 }, { 5, 0, 58 } }, + {RPL_BUG_MYSQL_37426, 37426, true, { 5, 1, 0 }, { 5, 1, 26 } }, + /* MariaDB bugs (MDEV-nnnnn). */ + {RPL_BUG_MDEV_29621, 29621, false, { 10, 11,1 }, { 10, 11,3 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 10,1 }, { 10, 10,4 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 9, 2 }, { 10, 9, 6 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 8, 4 }, { 10, 8, 8 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 7, 5 }, { 10, 7, 9 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 6, 9 }, { 10, 6, 13 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 5, 17 }, { 10, 5, 20 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 4, 26 }, { 10, 4, 29 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 3, 36 }, { 10, 3, 39 } }, +}; + + +/** + Detect, based on the version in format description event, any bugs on the + master that we have a work-around for on the slave. Calculate a bitmask + for this with a bit set for any detected bug. +*/ +void +Relay_log_info::calc_master_bug_bitmask(const Format_description_log_event * + fd_event) +{ + const Version &master_ver= fd_event->server_version_split; + uint all_size= + sizeof(versions_for_all_bugs)/sizeof(versions_for_all_bugs[0]); + rpl_bug_id_t bitmask= 0; + for (uint i= 0; i < all_size; i++) + { + const Version &introduced_in= versions_for_all_bugs[i].introduced_in; + const Version &fixed_in= versions_for_all_bugs[i].fixed_in; + if (introduced_in <= master_ver && fixed_in > master_ver) + { + bitmask|= (1 << versions_for_all_bugs[i].bug_id); + } + } + rpl_master_bug_bitmask.store(bitmask, std::memory_order_relaxed); +} + + +/** + Detects, based on version of master's format description event, if master + has a certain bug. The bitset of bugs was pre-calculated by + Relay_log_info::calc_master_bug_bitmask(). + + The bitset is read out of the Relay_log_info without locking. However this + is ok, as the set of master bugs can only change with a master server + restart, and we do not replicate in parallel events crossing a master + restart (as identified by its restart format description event). + + @param rli The Relay_log_info contains the pre-computed bug bitset. + @param bug_id Identifier of the bug as defined in enum_rpl_master_bug_ids. + @param report bool report error message. + + @param pred Predicate function that will be called with @c param to + check for the bug. If the function return @c true, the bug is present, + otherwise, it is not. + + @param param State passed to @c pred function. + + @return TRUE if master has the bug, FALSE if it does not. +*/ +bool +rpl_master_has_bug_ext(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, + bool (*pred)(const void *), const void *param) +{ + rpl_bug_id_t mask= + rli->rpl_master_bug_bitmask.load(std::memory_order_relaxed); + if (!(mask & (1 << bug_id))) + return false; + if (pred != NULL && !(*pred)(param)) + return false; + if (!report) + return true; + uint all_size= + sizeof(versions_for_all_bugs)/sizeof(versions_for_all_bugs[0]); + for (uint i= 0; i < all_size; i++) + { + const Version &fixed_in= versions_for_all_bugs[i].fixed_in; + if (versions_for_all_bugs[i].bug_id == bug_id) + { + const char *bug_source= versions_for_all_bugs[i].mysql_bug ? + "http://bugs.mysql.com/bug.php?id=" : + "https://jira.mariadb.org/browse/MDEV-"; + // a short message for SHOW SLAVE STATUS (message length constraints). + my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from" + " %s%u" + " so slave stops; check error log on slave" + " for more info", MYF(0), bug_source, + versions_for_all_bugs[i].bug_num); + // a verbose message for the error log. + rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, + "Master's version is earlier than %d.%d.%d," + " it is probable that master suffers from this bug:" + " %s%u" + " and thus replicating the current binary log event" + " may make the slave's data become different from the" + " master's data." + " To take no risk, slave refuses to replicate" + " this event and stops." + " We recommend that all updates be stopped on the" + " master and slave, that the data of both be" + " manually synchronized," + " that master's binary logs be deleted," + " that master be upgraded to a version at least" + " equal to '%d.%d.%d'. Then replication can be" + " restarted.", + fixed_in[0], fixed_in[1], fixed_in[2], + bug_source, + versions_for_all_bugs[i].bug_num, + fixed_in[0], fixed_in[1], fixed_in[2]); + return true; + } + } + DBUG_ASSERT(0 /* Impossible, as bitmap calculated from the same table. */); + my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from a serious bug" + " so slave stops", MYF(0)); + rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, + "Master's version is old," + " it is probable that master suffers from a serious bug" + " and thus replicating the current binary log event" + " may make the slave's data become different from the" + " master's data." + " To take no risk, slave refuses to replicate" + " this event and stops." + " We recommend that all updates be stopped on the" + " master and slave, that the data of both be" + " manually synchronized," + " that master's binary logs be deleted," + " that master be upgraded to a recent version" + " Then replication can be" + " restarted."); + return true; +} + #endif diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 9d21c1bf176..7f960c49c4a 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -282,6 +282,19 @@ public: */ bool are_sql_threads_caught_up(); + /** + Bitmask of bugs present in the master version of the current + description_event_for_exec. Of atomic type, since it is written when + applying master's format description events, and read concurrently without + locking from parallel replication worker threads. Though note that the + value will only change during a master restart, and we do not replicate + events concurrently across such restart event, so the value will not change + in practice while worker threads are working. + */ + std::atomic rpl_master_bug_bitmask; + static_assert(sizeof(rpl_bug_id_t) > ((RPL_BUG_END - 1) >> 3), + "rpl_master_bug_bitmask too small to hold all bits"); + void clear_until_condition(); /** Reset the delay. @@ -559,6 +572,7 @@ public: int32 get_sql_delay() { return sql_delay; } void set_sql_delay(int32 _sql_delay) { sql_delay= _sql_delay; } time_t get_sql_delay_end() { return sql_delay_end; } + void calc_master_bug_bitmask(const Format_description_log_event * fd_event); rpl_gtid last_seen_gtid; ulong last_trans_retry_count; private: @@ -1090,4 +1104,19 @@ int event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev); void delete_or_keep_event_post_apply(rpl_group_info *rgi, Log_event_type typ, Log_event *ev); +bool +rpl_master_has_bug_ext(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, + bool (*pred)(const void *), const void *param); +static inline bool +rpl_master_has_bug(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, bool (*pred)(const void *), const void *param) +{ + rpl_bug_id_t mask= + rli->rpl_master_bug_bitmask.load(std::memory_order_relaxed); + if (likely(!(mask & (1 << bug_id)))) + return false; + return rpl_master_has_bug_ext(rli, bug_id, report, pred, param); +} + #endif /* RPL_RLI_H */ diff --git a/sql/rpl_utility_server.cc b/sql/rpl_utility_server.cc index ccad7bd0709..de6ba19357f 100644 --- a/sql/rpl_utility_server.cc +++ b/sql/rpl_utility_server.cc @@ -775,9 +775,8 @@ static void show_sql_type(const Conv_source &src, const Field &dst, acceptable according to the current settings. @param order The computed order of the conversion needed. - @param rli The relay log info data structure: for error reporting. */ -static bool is_conversion_ok(enum_conv_type type, const Relay_log_info *rli, +static bool is_conversion_ok(enum_conv_type type, ulonglong type_conversion_options) { DBUG_ENTER("is_conversion_ok"); @@ -948,12 +947,10 @@ table_def::compatible_with(THD *thd, rpl_group_info *rgi, return false; } - if (!h) - return false; // An unknown data type found in the binary log Conv_source source(h, field_metadata(col), field->charset()); enum_conv_type convtype= can_convert_field_to(field, source, rli, Conv_param(m_flags)); - if (is_conversion_ok(convtype, rli, slave_type_conversions_options)) + if (is_conversion_ok(convtype, slave_type_conversions_options)) { DBUG_PRINT("debug", ("Checking column %d -" " field '%s' can be converted - order: %d", diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index 873675d6ea4..a403ac540fa 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -220,7 +220,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) { my_safe_printf_stderr("%s", "The information page at " - "https://mariadb.com/docs/server/reference/product-development/mariadb-fault-finding/how-to-produce-a-full-stack-trace-for-mariadbd" + "https://mariadb.com/docs/server/reference/product-development/mariadb-fault-finding/how-to-produce-a-full-stack-trace-for-mariadbd\n" "contains instructions to obtain a better version of the backtrace below.\n" "Following these instructions will help MariaDB developers provide a fix quicker.\n\n" "Attempting backtrace. Include this in the bug report.\n" diff --git a/sql/slave.cc b/sql/slave.cc index e41db54f473..54550a42eff 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -8323,104 +8323,6 @@ end: } -/** - Detects, based on master's version (as found in the relay log), if master - has a certain bug. - @param rli Relay_log_info which tells the master's version - @param bug_id Number of the bug as found in bugs.mysql.com - @param report bool report error message, default TRUE - - @param pred Predicate function that will be called with @c param to - check for the bug. If the function return @c true, the bug is present, - otherwise, it is not. - - @param param State passed to @c pred function. - - @return TRUE if master has the bug, FALSE if it does not. -*/ -bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, - bool (*pred)(const void *), const void *param, - bool maria_master) -{ - struct st_version_range_for_one_bug { - uint bug_id; - Version introduced_in; // first version with bug - Version fixed_in; // first version with fix - }; - static struct st_version_range_for_one_bug versions_for_their_bugs[]= - { - {24432, { 5, 0, 24 }, { 5, 0, 38 } }, - {24432, { 5, 1, 12 }, { 5, 1, 17 } }, - {33029, { 5, 0, 0 }, { 5, 0, 58 } }, - {33029, { 5, 1, 0 }, { 5, 1, 12 } }, - {37426, { 5, 1, 0 }, { 5, 1, 26 } }, - }; - static struct st_version_range_for_one_bug versions_for_our_bugs[]= - { - {29621, { 10, 3, 36 }, { 10, 3, 39 } }, - {29621, { 10, 4, 26 }, { 10, 4, 29 } }, - {29621, { 10, 5, 17 }, { 10, 5, 20 } }, - {29621, { 10, 6, 9 }, { 10, 6, 13 } }, - {29621, { 10, 7, 5 }, { 10, 7, 9 } }, - {29621, { 10, 8, 4 }, { 10, 8, 8 } }, - {29621, { 10, 9, 2 }, { 10, 9, 6 } }, - {29621, { 10, 10,1 }, { 10, 10,4 } }, - {29621, { 10, 11,1 }, { 10, 11,3 } }, - }; - const Version &master_ver= - rli->relay_log.description_event_for_exec->server_version_split; - struct st_version_range_for_one_bug* versions_for_all_bugs= maria_master ? - versions_for_our_bugs : versions_for_their_bugs; - uint all_size= maria_master ? - sizeof(versions_for_our_bugs)/sizeof(*versions_for_our_bugs) : - sizeof(versions_for_their_bugs)/sizeof(*versions_for_their_bugs); - - for (uint i= 0; i < all_size; i++) - { - const Version &introduced_in= versions_for_all_bugs[i].introduced_in; - const Version &fixed_in= versions_for_all_bugs[i].fixed_in; - if ((versions_for_all_bugs[i].bug_id == bug_id) && - introduced_in <= master_ver && - fixed_in > master_ver && - (pred == NULL || (*pred)(param))) - { - const char *bug_source= maria_master ? - "https://jira.mariadb.org/browse/MDEV-" : - "http://bugs.mysql.com/bug.php?id="; - if (!report) - return TRUE; - // a short message for SHOW SLAVE STATUS (message length constraints) - my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from" - " %s%u" - " so slave stops; check error log on slave" - " for more info", MYF(0), bug_source, bug_id); - // a verbose message for the error log - rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, - "According to the master's version ('%s')," - " it is probable that master suffers from this bug:" - " %s%u" - " and thus replicating the current binary log event" - " may make the slave's data become different from the" - " master's data." - " To take no risk, slave refuses to replicate" - " this event and stops." - " We recommend that all updates be stopped on the" - " master and slave, that the data of both be" - " manually synchronized," - " that master's binary logs be deleted," - " that master be upgraded to a version at least" - " equal to '%d.%d.%d'. Then replication can be" - " restarted.", - rli->relay_log.description_event_for_exec->server_version, - bug_source, - bug_id, - fixed_in[0], fixed_in[1], fixed_in[2]); - return TRUE; - } - } - return FALSE; -} - /** BUG#33029, For all 5.0 up to 5.0.58 exclusive, and 5.1 up to 5.1.12 exclusive, if one statement in a SP generated AUTO_INCREMENT value @@ -8436,7 +8338,8 @@ bool rpl_master_erroneous_autoinc(THD *thd) if (thd->rgi_slave) { DBUG_EXECUTE_IF("simulate_bug33029", return TRUE;); - return rpl_master_has_bug(thd->rgi_slave->rli, 33029, FALSE, NULL, NULL); + return rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_33029, FALSE, + NULL, NULL); } return FALSE; } diff --git a/sql/slave.h b/sql/slave.h index ce25bebf730..c5670447d09 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -230,9 +230,6 @@ bool show_master_info(THD* thd, Master_info* mi, bool full); bool show_all_master_info(THD* thd); void show_binlog_info_get_fields(THD *thd, List *field_list); bool show_binlog_info(THD* thd); -bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, - bool (*pred)(const void *), const void *param, - bool maria_master= false); bool rpl_master_erroneous_autoinc(THD* thd); const char *print_slave_db_safe(const char *db); diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 9dc02125dfe..ecf9b8ef11d 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -582,6 +582,7 @@ sp_head::sp_head(MEM_ROOT *mem_root_arg, sp_package *parent, m_thd_root(NULL), m_thd(NULL), m_pcont(new (&main_mem_root) sp_pcontext()), + m_pcont_list(NULL), m_cont_level(0) { mem_root= &main_mem_root; @@ -902,6 +903,17 @@ sp_head::~sp_head() for (uint ip = 0 ; (i = get_instr(ip)) ; ip++) delete i; delete_dynamic(&m_instr); + + /* + Free the parse contexts iteratively, freeing them recursively overran + the thread stack on deeply nested BEGIN blocks. + */ + for (sp_pcontext *ctx= m_pcont_list; ctx; ) + { + sp_pcontext *next= ctx->m_next_pcont; + delete ctx; + ctx= next; + } delete m_pcont; free_items(); @@ -1712,7 +1724,8 @@ sp_rcontext *sp_head::rcontext_create(THD *thd, Field *ret_value, { DBUG_ASSERT(args); Row_definition_list defs; - m_pcont->retrieve_field_definitions(&defs); + if (m_pcont->retrieve_field_definitions(thd, &defs)) + return NULL; if (defs.adjust_formal_params_to_actual_params(thd, args)) return NULL; return rcontext_create(thd, ret_value, &defs, true); @@ -1723,7 +1736,8 @@ sp_rcontext *sp_head::rcontext_create(THD *thd, Field *ret_value, Item **args, uint arg_count) { Row_definition_list defs; - m_pcont->retrieve_field_definitions(&defs); + if (m_pcont->retrieve_field_definitions(thd, &defs)) + return NULL; if (defs.adjust_formal_params_to_actual_params(thd, args, arg_count)) return NULL; return rcontext_create(thd, ret_value, &defs, true); @@ -1828,8 +1842,8 @@ sp_head::execute_trigger(THD *thd, thd->set_n_backup_active_arena(&call_arena, &backup_arena); Row_definition_list defs; - m_pcont->retrieve_field_definitions(&defs); - if (!(nctx= rcontext_create(thd, NULL, &defs, false))) + if (m_pcont->retrieve_field_definitions(thd, &defs) || + !(nctx= rcontext_create(thd, NULL, &defs, false))) { err_status= TRUE; goto err_with_cleanup; diff --git a/sql/sp_head.h b/sql/sp_head.h index f87893a6727..3566ab3e31b 100644 --- a/sql/sp_head.h +++ b/sql/sp_head.h @@ -938,6 +938,16 @@ public: sp_pcontext *get_parse_context() { return m_pcont; } + /* + Remember a parse context of this routine, so that ~sp_head frees the + whole tree iteratively instead of recursing down sp_pcontext's children + */ + void remember_pcontext(sp_pcontext *ctx) + { + ctx->m_next_pcont= m_pcont_list; + m_pcont_list= ctx; + } + /* Check EXECUTE access: - in case of a standalone rotuine, for the routine itself @@ -958,6 +968,7 @@ protected: THD *m_thd; ///< Set if we have reset mem_root sp_pcontext *m_pcont; ///< Parse context + sp_pcontext *m_pcont_list; ///< Head of context list, freed by ~sp_head List m_lex; ///< Temp. store for the other lex DYNAMIC_ARRAY m_instr; ///< The "instructions" diff --git a/sql/sp_pcontext.cc b/sql/sp_pcontext.cc index 4bb22838578..8ca115e53f8 100644 --- a/sql/sp_pcontext.cc +++ b/sql/sp_pcontext.cc @@ -23,6 +23,7 @@ #include "sp_pcontext.h" #include "sp_head.h" +#include "sql_parse.h" bool sp_condition_value::equals(const sp_condition_value *cv) const { @@ -95,7 +96,7 @@ void sp_pcontext::init(uint var_offset, sp_pcontext::sp_pcontext() : Sql_alloc(), m_max_var_index(0), m_max_cursor_index(0), - m_parent(NULL), m_pboundary(0), + m_parent(NULL), m_next_pcont(NULL), m_pboundary(0), m_vars(PSI_INSTRUMENT_MEM), m_case_expr_ids(PSI_INSTRUMENT_MEM), m_conditions(PSI_INSTRUMENT_MEM), m_cursors(PSI_INSTRUMENT_MEM), m_handlers(PSI_INSTRUMENT_MEM), m_children(PSI_INSTRUMENT_MEM), @@ -108,7 +109,7 @@ sp_pcontext::sp_pcontext() sp_pcontext::sp_pcontext(sp_pcontext *prev, sp_pcontext::enum_scope scope) : Sql_alloc(), m_max_var_index(0), m_max_cursor_index(0), - m_parent(prev), m_pboundary(0), + m_parent(prev), m_next_pcont(NULL), m_pboundary(0), m_vars(PSI_INSTRUMENT_MEM), m_case_expr_ids(PSI_INSTRUMENT_MEM), m_conditions(PSI_INSTRUMENT_MEM), m_cursors(PSI_INSTRUMENT_MEM), m_handlers(PSI_INSTRUMENT_MEM), m_children(PSI_INSTRUMENT_MEM), @@ -120,19 +121,16 @@ sp_pcontext::sp_pcontext(sp_pcontext *prev, sp_pcontext::enum_scope scope) } -sp_pcontext::~sp_pcontext() -{ - for (size_t i= 0; i < m_children.elements(); ++i) - delete m_children.at(i); -} - - -sp_pcontext *sp_pcontext::push_context(THD *thd, sp_pcontext::enum_scope scope) +sp_pcontext *sp_pcontext::push_context(THD *thd, sp_head *sphead, + sp_pcontext::enum_scope scope) { sp_pcontext *child= new (thd->mem_root) sp_pcontext(this, scope); if (child) + { m_children.append(child); + sphead->remember_pcontext(child); + } return child; } @@ -644,9 +642,12 @@ const sp_pcursor *sp_pcontext::find_cursor(const LEX_CSTRING *name, } -void sp_pcontext::retrieve_field_definitions( - List *field_def_lst) const +bool sp_pcontext::retrieve_field_definitions( + THD *thd, List *field_def_lst) const { + if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) + return true; + size_t next_child_idx= 0; size_t next_variable_idx= 0; @@ -656,15 +657,16 @@ void sp_pcontext::retrieve_field_definitions( { // No variables left, just retrieve the remaining children contexts for (size_t i= next_child_idx; i < m_children.elements(); ++i) - m_children.at(i)->retrieve_field_definitions(field_def_lst); - return; + if (m_children.at(i)->retrieve_field_definitions(thd, field_def_lst)) + return true; + return false; } if (next_child_idx >= m_children.elements()) { // No child contexts left, just retrieve the remaining variables for (size_t i= next_variable_idx; i < m_vars.elements(); ++i) field_def_lst->push_back(&m_vars.at(i)->field_def); - return; + return false; } sp_variable *next_variable= m_vars.at(next_variable_idx); @@ -713,11 +715,13 @@ void sp_pcontext::retrieve_field_definitions( DBUG_ASSERT(next_child_idx + 1 < m_children.elements() ? next_child->after_last_variable_offset() <= next_variable->offset : next_child->after_last_variable_offset() == next_variable->offset); - next_child->retrieve_field_definitions(field_def_lst); + if (next_child->retrieve_field_definitions(thd, field_def_lst)) + return true; next_child_idx++; } } + return false; } diff --git a/sql/sp_pcontext.h b/sql/sp_pcontext.h index 265e1dbf976..410f20b3100 100644 --- a/sql/sp_pcontext.h +++ b/sql/sp_pcontext.h @@ -378,8 +378,12 @@ public: /// - for error checking (e.g. to check correct number of parameters); /// - to resolve SQL-handlers. +class sp_head; + class sp_pcontext : public Sql_alloc { + friend class sp_head; + public: enum enum_scope { @@ -419,15 +423,19 @@ public: public: sp_pcontext(); - ~sp_pcontext(); + /* + No destructor: sp_head owns the tree and frees it. Freeing the children + here recursed and overran the thread stack on deeply nested BEGIN blocks + */ /// Create and push a new context in the tree. - /// @param thd thread context. - /// @param scope scope of the new parsing context. + /// @param thd thread context. + /// @param sphead the routine owning the tree, which remembers the new node. + /// @param scope scope of the new parsing context. /// @return the node created. - sp_pcontext *push_context(THD *thd, enum_scope scope); + sp_pcontext *push_context(THD *thd, sp_head *sphead, enum_scope scope); /// Pop a node from the parsing context tree. /// @return the parent node. @@ -524,8 +532,11 @@ public: /// Retrieve full type information about SP-variables in this parsing /// context and its children. /// + /// @param thd thread context. /// @param field_def_lst[out] Container to store type information. - void retrieve_field_definitions(List *field_def_lst) const; + /// @return true on error, in which case field_def_lst is incomplete. + bool retrieve_field_definitions(THD *thd, + List *field_def_lst) const; /// Find SP-variable by name. /// @@ -773,6 +784,9 @@ private: /// Parent context. sp_pcontext *m_parent; + /// Next context in the sp_head's single linked list of all contexts. + sp_pcontext *m_next_pcont; + /// An index of the first SP-variable in this parsing context. The index /// belongs to a runtime table of SP-variables. /// diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index cfef5103a05..88ac9515a60 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -944,7 +944,8 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_24432, + TRUE, NULL, NULL)) goto abort; #endif @@ -4338,7 +4339,8 @@ select_insert::prepare(List &values, SELECT_LEX_UNIT *u) if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_24432, + TRUE, NULL, NULL)) DBUG_RETURN(1); #endif @@ -5401,8 +5403,12 @@ bool select_create::send_eof() if (!table->s->tmp_table) { #ifdef WITH_WSREP + // For partitioned tables need to look implementing hton + const handlerton* ht= table->file->partition_ht() ? + table->file->partition_ht() : + table->file->ht; if (WSREP(thd) && - table->file->ht->db_type == DB_TYPE_INNODB) + ht->db_type == DB_TYPE_INNODB) { if (thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID) { diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 5a54ffb4314..7c5e1f12815 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -7460,7 +7460,7 @@ bool LEX::sp_handler_declaration_init(THD *thd, int type) { sp_handler *h= spcont->add_handler(thd, (sp_handler::enum_type) type); - spcont= spcont->push_context(thd, sp_pcontext::HANDLER_SCOPE); + spcont= spcont->push_context(thd, sphead, sp_pcontext::HANDLER_SCOPE); sp_instr_hpush_jump *i= new (thd->mem_root) sp_instr_hpush_jump(sphead->instructions(), spcont, h); @@ -7511,7 +7511,7 @@ bool LEX::sp_handler_declaration_finalize(THD *thd, int type) void LEX::sp_block_init(THD *thd, const LEX_CSTRING *label) { spcont->push_label(thd, label, sphead->instructions(), sp_label::BEGIN); - spcont= spcont->push_context(thd, sp_pcontext::REGULAR_SCOPE); + spcont= spcont->push_context(thd, sphead, sp_pcontext::REGULAR_SCOPE); } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index e4cab9e096a..fe304f710e9 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -17990,6 +17990,10 @@ return_zero_rows(JOIN *join, select_result *result, List *tables, { join->thd->limit_found_rows= 0; } + else + { + join->thd->limit_found_rows= send_row ? 1 : 0; + } if (!(result->send_result_set_metadata(*fields, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF))) @@ -26027,6 +26031,7 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) join->fields. */ List *fields= join_tab ? (join_tab-1)->fields : join->fields; + bool empty_set_send_rollup_total= false; DBUG_ENTER("end_send_group"); if (!join->items3.is_null() && join->current_ref_ptrs != join->items3) @@ -26038,7 +26043,8 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) if (!join->group_sent && (join->first_record || - (end_of_records && !join->group && !join->group_optimized_away))) + join->need_empty_set_row(end_of_records, + &empty_set_send_rollup_total))) { table_map cleared_tables= (table_map) 0; if (join->procedure) @@ -26094,7 +26100,8 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) join->send_records++; join->group_sent= true; } - if (unlikely(join->rollup.state != ROLLUP::STATE_NONE && error <= 0)) + if (unlikely(join->rollup.state != ROLLUP::STATE_NONE && + !empty_set_send_rollup_total && error <= 0)) { if (join->rollup_send_data((uint) (idx+1))) error= 1; @@ -26414,13 +26421,15 @@ end_write_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), { TABLE *table= join_tab->table; int idx= -1; + bool empty_set_send_rollup_total= false; DBUG_ENTER("end_write_group"); join->accepted_rows++; if (!join->first_record || end_of_records || (idx=test_if_group_changed(join->group_fields)) >= 0) { - if (join->first_record || (end_of_records && !join->group)) + if (join->first_record || + join->need_empty_set_row(end_of_records, &empty_set_send_rollup_total)) { table_map cleared_tables= (table_map) 0; if (join->procedure) @@ -26445,7 +26454,8 @@ end_write_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), error, 0, NULL)) DBUG_RETURN(NESTED_LOOP_ERROR); } - if (unlikely(join->rollup.state != ROLLUP::STATE_NONE)) + if (unlikely(join->rollup.state != ROLLUP::STATE_NONE && + !empty_set_send_rollup_total)) { if (unlikely(join->rollup_write_data((uint) (idx+1), join_tab->tmp_table_param, diff --git a/sql/sql_select.h b/sql/sql_select.h index ea1f51b51ca..2711b1138b1 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1837,12 +1837,27 @@ public: void cleanup(bool full); void clear(table_map *cleared_tables); void inline clear_sum_funcs(); + bool need_empty_set_row(bool end_of_records, + bool *empty_set_send_rollup_total) const + { + *empty_set_send_rollup_total= + end_of_records && !first_record && rollup.state != ROLLUP::STATE_NONE; + return (end_of_records && !first_record && !group && + !group_optimized_away) || + *empty_set_send_rollup_total; + } bool send_row_on_empty_set() { - return (do_send_rows && implicit_grouping && !group_optimized_away && + return (do_send_rows && + (select_lex->olap == ROLLUP_TYPE || + (implicit_grouping && !group_optimized_away)) && having_value != Item::COND_FALSE); } - bool empty_result() { return (zero_result_cause && !implicit_grouping); } + bool empty_result() + { + return (zero_result_cause && !implicit_grouping && + select_lex->olap != ROLLUP_TYPE); + } bool change_result(select_result *new_result, select_result *old_result); bool is_top_level_join() const { diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index dc9809ad526..c70cfa92282 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -120,11 +120,10 @@ my_bool wsrep_incremental_data_collection= 0; // Incremental data collection bool wsrep_new_cluster= false; // Bootstrap the cluster? int wsrep_slave_count_change= 0; // No. of appliers to stop/start int wsrep_to_isolation= 0; // No. of active TO isolation threads -// -// NOTE : MySQL Cluster has max_protocol version 7 -// thus protocol versions 5-7 are reserved for compatibility. +// 5: CHAR write set key values are normalized to the number of characters +// the column holds, see wsrep_store_string_key_val() +long wsrep_max_protocol_version= 5; // Maximum protocol version to use // -long wsrep_max_protocol_version= 4; // Maximum protocol version to use long int wsrep_protocol_version= wsrep_max_protocol_version; ulong wsrep_trx_fragment_unit= WSREP_FRAG_BYTES; // unit for fragment size @@ -2009,7 +2008,6 @@ static bool wsrep_prepare_key_for_isolation(const char* db, case 2: case 3: case 4: - // For MySQL Galera Cluster migration case 5: case 6: case 7: @@ -2192,7 +2190,6 @@ bool wsrep_prepare_key(const uchar* cache_key, size_t cache_key_len, case 2: case 3: case 4: - // For MySQL Galera Cluster migration case 5: case 6: case 7: diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index b8f220768a0..aa578f0e176 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -2890,8 +2890,12 @@ fts_query_phrase_search( func_exit: mem_heap_free(heap); - /* Don't need it anymore. */ + /* Don't need it anymore. + Reset the phrase/proximity flag to avoid the subsequent + wildcard term in the same boolean query would take the + exact word lookup path in fts_query_cache() */ query->matched = NULL; + query->flags = 0; return(query->error); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 623b912b677..7907cd43d7b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6151,6 +6151,37 @@ ha_innobase::close() /* The following accessor functions should really be inside MySQL code! */ #ifdef WITH_WSREP +/** Pick the charset struct of a charset number, for the write set key +encoding. Since the MySQL function get_charset may be slow before Bar +removes the mutex operation there, we first look at 2 common charsets +directly. +@param charset_number number of the charset +@return the charset, never NULL */ +static +CHARSET_INFO* +wsrep_get_charset(uint charset_number) +{ + if (charset_number == default_charset_info->number) { + return default_charset_info; + } + + if (charset_number == my_charset_latin1.number) { + return &my_charset_latin1; + } + + CHARSET_INFO* charset = get_charset(charset_number, MYF(MY_WME)); + + if (charset == NULL) { + sql_print_error("InnoDB needs charset %lu for doing " + "a comparison, but MariaDB cannot " + "find that charset.", + (ulong) charset_number); + ut_a(0); + } + + return charset; +} + size_t wsrep_normalize_string( int mysql_type, /* in: MySQL type */ @@ -6178,28 +6209,7 @@ wsrep_normalize_string( case MYSQL_TYPE_LONG_BLOB: case MYSQL_TYPE_VARCHAR: { - CHARSET_INFO* charset; - - /* Use the charset number to pick the right charset struct for - the comparison. Since the MySQL function get_charset may be - slow before Bar removes the mutex operation there, we first - look at 2 common charsets directly. */ - - if (charset_number == default_charset_info->number) { - charset = default_charset_info; - } else if (charset_number == my_charset_latin1.number) { - charset = &my_charset_latin1; - } else { - charset = get_charset(charset_number, MYF(MY_WME)); - - if (charset == NULL) { - sql_print_error("InnoDB needs charset %lu for doing " - "a comparison, but MariaDB cannot " - "find that charset.", - (ulong) charset_number); - ut_a(0); - } - } + CHARSET_INFO* charset = wsrep_get_charset(charset_number); if (wsrep_protocol_version < 3) { ret_length = charset->strnxfrm( @@ -6243,6 +6253,122 @@ wsrep_normalize_string( return ret_length; } + +/** Store the write set key value of a string column. + +Both write set key paths come here, so that one and the same column value +always produces one and the same key: wsrep_store_key_val_for_row() has the +value in the MySQL record format and wsrep_rec_get_foreign_key() has it as +InnoDB stored it, and a CHAR is not padded the same way in the two. + +From protocol version 5 on, the MySQL record pads a CHAR to +n_chars * mbmaxlen bytes, while InnoDB in a compact row format and a +variable length character set strips that padding down to but not below +n_chars bytes; as that compares a byte count against a character count, a +value holding multi byte characters is left with fewer than n_chars +characters. See row_mysql_store_col_in_innobase_format(). Both forms are +brought to exactly n_chars characters here, which also makes the key +independent of the row format. The normalization itself is always done with +WSREP_MAX_SUPPORTED_KEY_LENGTH as the buffer length, and only the copy into +out_str is limited by the space the caller has left, because strnxfrm +truncates to the length it is given and the key of a column must not depend +on how much room the columns before it happened to leave. + +Before protocol version 5 the two paths did neither of those and disagreed +for a CHAR in a variable length character set. That is kept here so that a +node still speaking the older protocol produces the same keys as before. +Note that the old paths did not agree on the strnxfrm buffer length either, +3072 on one and 3500 on the other; both get WSREP_MAX_SUPPORTED_KEY_LENGTH +here, which only makes a difference for a value whose normalized form is +longer than 3072 bytes, where the two never produced the same key anyway. + +@param mysql_type MySQL type of the column +@param charset_number number of the charset of the column +@param n_chars characters the column holds, 0 if it is not a CHAR +@param str column value +@param str_length length of str in bytes +@param out_str buffer for the key value +@param out_length space left in out_str +@param mysql_format str is in the MySQL record format, where a CHAR is + padded to more characters than the column holds; + before protocol 5 that was cut back here +@return number of bytes written to out_str */ +size_t +wsrep_store_string_key_val( + int mysql_type, + uint charset_number, + size_t n_chars, + const unsigned char* str, + size_t str_length, + unsigned char* out_str, + ulint out_length, + bool mysql_format) +{ + unsigned char normalized[WSREP_MAX_SUPPORTED_KEY_LENGTH + 1]; + size_t len; + + if (wsrep_protocol_version < 5) { + CHARSET_INFO* cs = wsrep_get_charset(charset_number); + + if (mysql_format && str_length > 0 && cs->mbmaxlen > 1) { + int error; + + str_length = my_well_formed_length( + cs, (const char*) str, + (const char*) str + str_length, + str_length / cs->mbmaxlen, &error); + } + + len = wsrep_normalize_string(mysql_type, charset_number, str, + normalized, str_length, + WSREP_MAX_SUPPORTED_KEY_LENGTH); + } else { + unsigned char padded[REC_VERSION_56_MAX_INDEX_COL_LEN + 1]; + + if (n_chars) { + CHARSET_INFO* cs = wsrep_get_charset(charset_number); + int error; + + /* Cut to at most n_chars characters ... */ + str_length = my_well_formed_length( + cs, (const char*) str, + (const char*) str + str_length, n_chars, + &error); + + /* ... and pad back up to n_chars where InnoDB had + stripped the padding. mbminlen is 1 for every + character set whose CHAR padding InnoDB strips, so + the pad character is one 0x20. */ + const size_t chars = cs->numchars( + (const char*) str, + (const char*) str + str_length); + + if (chars < n_chars) { + const size_t pad = n_chars - chars; + + ut_ad(cs->mbminlen == 1); + ut_a(str_length + pad <= sizeof padded); + + memcpy(padded, str, str_length); + memset(padded + str_length, 0x20, pad); + str = padded; + str_length += pad; + } + } + + len = wsrep_normalize_string(mysql_type, charset_number, str, + normalized, str_length, + WSREP_MAX_SUPPORTED_KEY_LENGTH); + } + + if (len > out_length) { + len = out_length; + } + + memcpy(out_str, normalized, len); + + return len; +} #endif /* WITH_WSREP */ /******************************************************************//** @@ -6814,38 +6940,26 @@ wsrep_store_key_val_for_row( const CHARSET_INFO* cs= field->charset(); - /* For multi byte character sets we need to - calculate the true length of the key */ - - if (true_len > 0 && cs->mbmaxlen > 1) { - int error; - - true_len= my_well_formed_length(cs, - (const char *)src_start, - (const char *)src_start - + true_len, - (true_len / cs->mbmaxlen), - &error); - } + /* Only a whole CHAR column is brought to the + number of characters it holds. A prefix key + part is not a column value and is never + matched against a foreign key. */ + const size_t n_chars= + (mysql_type == MYSQL_TYPE_STRING + && key_part->length + == field->pack_length()) + ? key_part->length / cs->mbmaxlen : 0; /* Normalize string if it is not empty string */ if (true_len) { ut_ad(src_start); - true_len= wsrep_normalize_string( - mysql_type, cs->number, - src_start, normalized, true_len, - REC_VERSION_56_MAX_INDEX_COL_LEN); + true_len= wsrep_store_string_key_val( + mysql_type, cs->number, n_chars, + src_start, true_len, + (uchar*) buff, buff_space, true); } else { ut_ad(src_start == nullptr); } - - if (true_len > buff_space) { - fprintf (stderr, - "WSREP: key truncated: %s\n", - wsrep_thd_query(thd)); - true_len = buff_space; - } - memcpy(buff, normalized, true_len); } else { /* Copy only if there is data */ if (true_len) { diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index b3d94d55956..0fba3dd8400 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -4311,20 +4311,19 @@ created_clustered: if (add_fts_doc_idx) { index_def_t* index = indexdef++; - uint nfields = 1; - - if (altered_table->versioned()) - ++nfields; + const uint nfields = altered_table->versioned() + 1; index->fields = static_cast( - mem_heap_alloc(heap, sizeof(*index->fields) * nfields)); + mem_heap_alloc(heap, nfields * sizeof *index->fields)); index->n_fields = nfields; index->fields[0].col_no = fts_doc_id_col; index->fields[0].prefix_len = 0; index->fields[0].descending = false; index->fields[0].is_v_col = false; if (nfields == 2) { - index->fields[1].col_no - = altered_table->s->vers.end_fieldno; + /* FTS_DOC_ID_INDEX(FTS_DOC_ID,row_end) */ + index->fields[1].col_no = innodb_col_no( + altered_table->field[ + altered_table->s->vers.end_fieldno]); index->fields[1].prefix_len = 0; index->fields[1].descending = false; index->fields[1].is_v_col = false; diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 169173dd9c7..3af619b5499 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -154,6 +154,18 @@ size_t wsrep_normalize_string(int mysql_type, unsigned char* out_str, ulint str_length, ulint buf_length); + +/** Store the write set key value of a string column. Used by both write set +key paths, so that one and the same column value always produces one and the +same key. See the definition in ha_innodb.cc. */ +size_t wsrep_store_string_key_val(int mysql_type, + uint charset_number, + size_t n_chars, + const unsigned char* str, + size_t str_length, + unsigned char* out_str, + ulint out_length, + bool mysql_format); #endif /* WITH_WSREP */ /** Get high resolution timestamp for the current query start time. diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index b0ec745d159..cb35073f487 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -486,9 +486,6 @@ public: /** @return purge_sys.view or purge_sys.end_view */ inline const ReadViewBase &view() const; - - /** @return whether this is part of CHECK TABLE ... EXTENDED */ - bool is_extended() const noexcept { return latch < END_VIEW; } }; struct end_view_guard diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 95a7faa1299..0011075ea21 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -2760,12 +2760,26 @@ wsrep_rec_get_foreign_key( } case DATA_VARCHAR: case DATA_VARMYSQL: - case DATA_CHAR: - case DATA_MYSQL: - len = wsrep_normalize_string( + len = wsrep_store_string_key_val( (int)(col_f->prtype & DATA_MYSQL_TYPE_MASK), dtype_get_charset_coll(col_f->prtype), - data, buf, len, *buf_len); + 0, data, len, buf, + *buf_len - key_len, false); + break; + case DATA_CHAR: + case DATA_MYSQL: + /* A CHAR is stored with the space padding + InnoDB happened to leave on it, which is not + what the MySQL record holds. Pass the number + of characters the column has so that both + forms end up as the same key. */ + len = wsrep_store_string_key_val( + (int)(col_f->prtype & DATA_MYSQL_TYPE_MASK), + dtype_get_charset_coll(col_f->prtype), + col_f->mbmaxlen + ? col_f->len / col_f->mbmaxlen : 0, + data, len, buf, + *buf_len - key_len, false); break; case DATA_BLOB: case DATA_BINARY: diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 291d3531b28..dd667eafce4 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -4013,6 +4013,24 @@ void UndorecApplier::log_update(const dtuple_t &tuple, rec_offs_make_valid(copy_rec, clust_index, true, offsets); mtr.commit(); + /* row_log_table_get_pk() below dereferences the externally stored + columns of prev_version, with neither a page latch nor a frozen + purge_sys.view. Two properties keep those pages allocated: + + (a) trx_t::apply_log() runs before trx_t::write_serialisation_history(), + so this transaction is still registered in trx_sys and its undo log is + not yet in the history list. purge_sys.view therefore cannot see it. + + (b) get_old_rec() stops at the version this undo log record wrote, and + this transaction holds an exclusive lock on the record. The page latch + is already released here, but that lock is not, until past the + serialisation point (a) rests on, so every version above that one is + this transaction's own. The oldest writer applied is therefore this + transaction, and whatever disowned a reference that prev_version holds + is that writer or a newer one, none of which purge can reach. + + Deferring the call past the serialisation, or walking below this + transaction's own version, would break the argument. */ clust_index->lock.s_lock(SRW_LOCK_CALL); /* Recheck whether clustered index online log has been cleared */ if (clust_index->online_log) diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 95a77e215b4..1991dbb8705 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -686,7 +686,8 @@ nochange_index: rec_offs_comp(clust_offsets))) { row_ext_t* ext; - /* The stack of versions is locked by mtr. + /* The stack of versions is locked by mtr, and + purge_sys.view is frozen for the whole batch. Thus, it is safe to fetch the prefixes for externally stored columns. */ row = row_build(ROW_COPY_POINTERS, clust_index, diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index dc7bf1b55b5..f56810e9da2 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -6541,24 +6541,63 @@ rec_loop: check_latest_version: /* In CHECK TABLE...EXTENDED, always check if the secondary index record matches the latest clustered index record - version, no matter if it is visible in our own read view. - - If the latest clustered index version is delete-marked and - purgeable, it is not safe to fetch any BLOBs for column prefix - indexes because they may already have been freed. */ - if (rec_trx_id && - rec_get_deleted_flag(clust_rec, - prebuilt->table->not_redundant()) && - purge_sys.is_purgeable(rec_trx_id)) - goto did_not_find; - + version, no matter if it is visible in our own read view. */ if (!clust_offsets) clust_offsets= rec_get_offsets(clust_rec, clust_index, nullptr, clust_index->n_core_fields, ULINT_UNDEFINED, &heap); - err= row_check_index_match(prebuilt, - clust_rec, clust_index, clust_offsets, - rec, index, offsets); + + /* If the latest clustered index version is delete-marked and + purgeable, it is not safe to fetch any BLOBs for column prefix + indexes because they may already have been freed. Where anything + will be dereferenced, freeze purge_sys.view across the test and + the fetch, so that purge cannot start freeing in between. + + The freeze deliberately spans the whole comparison, not only the + fetch: row_check_index_match() dereferences one field at a time, + and for an index over a virtual column it evaluates the column + expression as well. Purge stays blocked for as long as that + takes. + + A record that is not delete-marked owns all its externally stored + columns, and purge frees no reference that a record still owns, so + only a delete-marked one needs the freeze. The delete-mark cannot + change while this mini-transaction holds the page latch. + + DB_TRX_ID stays out of that choice. It is 0 on a delete-marked + record that IMPORT TABLESPACE could not remove, and such a record + still needs the freeze, its references having been disowned to a + version that purge may free. Where it is 0 there is no transaction + to ask about, which is what both tests below need. */ + { + const bool deleted= + rec_get_deleted_flag(clust_rec, prebuilt->table->not_redundant()); + + if (!deleted || !rec_offs_any_extern(clust_offsets) || + DBUG_IF("purge_no_blob_freeze")) + { + /* purge_no_blob_freeze sends a record that does have externally + stored columns here as well, which is how the fetch below behaved + before it was protected. */ + if (deleted && rec_trx_id && purge_sys.is_purgeable(rec_trx_id)) + goto did_not_find; + + err= row_check_index_match(prebuilt, + clust_rec, clust_index, clust_offsets, + rec, index, offsets); + } + else + { + purge_sys_t::view_guard freeze{purge_sys_t::view_guard::VIEW}; + + if (rec_trx_id && freeze.view().changes_visible(rec_trx_id)) + goto did_not_find; + + err= row_check_index_match(prebuilt, + clust_rec, clust_index, clust_offsets, + rec, index, offsets); + } + } switch (err) { default: @@ -6630,6 +6669,9 @@ rec_loop: for (;;) { + /* The writer of clust_rec, whose undo log record rebuilds the next + version. Read before clust_rec is replaced by it below. */ + const trx_id_t version_trx_id= rec_trx_id; mem_heap_t *prev_heap= vers_heap; vers_heap= mem_heap_create(1024); err= trx_undo_prev_version_build(clust_rec, @@ -6698,13 +6740,67 @@ rec_loop: if (&view != &trx->read_view) { /* It is not safe to fetch BLOBs of committed delete-marked - records that may have been freed in purge. */ - err= clust_rec_deleted && rec_trx_id && - purge_sys.is_purgeable(rec_trx_id) - ? DB_SUCCESS_LOCKED_REC - : row_check_index_match(prebuilt, - clust_rec, clust_index, clust_offsets, - rec, index, offsets); + records that may have been freed in purge. Where anything will + be dereferenced, freeze purge_sys.view across the test and the + fetch, so that purge cannot start freeing in between. As at + check_latest_version, the freeze deliberately spans the whole + comparison, which fetches one field at a time and may evaluate + a virtual column expression. */ + const bool extern_cols= rec_offs_any_extern(clust_offsets); + + if (!extern_cols || DBUG_IF("purge_no_blob_freeze")) + { + /* purge_no_blob_freeze sends a version that does have + externally stored columns here as well, which is how the + fetch below behaved before it was protected. */ + if (extern_cols) + DEBUG_SYNC_C("row_check_index_extended_match"); + err= clust_rec_deleted && rec_trx_id && + purge_sys.is_purgeable(rec_trx_id) + ? DB_SUCCESS_LOCKED_REC + : row_check_index_match(prebuilt, + clust_rec, clust_index, clust_offsets, + rec, index, offsets); + } + else + { + purge_sys_t::view_guard freeze{purge_sys_t::view_guard::VIEW}; + + /* Confirm that purge cannot see the writer of the version + clust_rec was rebuilt from. Unlike every other walk over old + versions, this one decides reachability from the lagging + purge_sys.end_view, so it does reach history that purge is + already free to remove. + + Where it does, stop and report nothing. The orphan reports + of the other exits are calibrated on purge_sys.end_view, + while this test speaks about purge_sys.view, which is ahead + of it. Feeding a decision of the one into a gate of the + other could raise an error for a record that is not + delete-marked and is nonetheless sound. No report is lost + for good: as soon as end_view reaches the same point, + trx_undo_prev_version_build() reports missing history, and + a record that is truly unmatchable is flagged then. + + row_check_index_purgeable forces this branch, which is + otherwise reached only when the view advances between the + walk starting and the freeze. */ + if (freeze.view().changes_visible(version_trx_id) || + DBUG_IF("row_check_index_purgeable")) + { + ut_ad(!found_in_view); + goto free_vers_heap; + } + + DEBUG_SYNC_C("row_check_index_extended_match"); + + err= clust_rec_deleted && rec_trx_id && + freeze.view().changes_visible(rec_trx_id) + ? DB_SUCCESS_LOCKED_REC + : row_check_index_match(prebuilt, + clust_rec, clust_index, clust_offsets, + rec, index, offsets); + } switch (err) { default: @@ -6722,6 +6818,7 @@ rec_loop: visible_trx_id= rec_trx_id; found_in_view= !clust_rec_deleted; } + free_vers_heap: mem_heap_free(vers_heap); if (!found_in_view) goto did_not_find; diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index cd30090d437..8161d8cd987 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -520,6 +520,11 @@ static bool row_undo_mod_sec_is_unsafe(const rec_t *rec, dict_index_t *index, version = rec; for (;;) { + /* The writer of version, whose undo log record rebuilds the + next one. Read before the heap holding version is freed. */ + const trx_id_t version_trx_id = row_get_rec_trx_id( + version, clust_index, clust_offsets); + heap2 = heap; heap = mem_heap_create(1024); vrow = NULL; @@ -566,12 +571,54 @@ nochange_index: if (!rec_get_deleted_flag(prev_version, comp)) { row_ext_t* ext; - /* The stack of versions is locked by mtr. - Thus, it is safe to fetch the prefixes for - externally stored columns. */ - row = row_build(ROW_COPY_POINTERS, clust_index, - prev_version, clust_offsets, - NULL, NULL, NULL, &ext, heap); + const bool extern_cols = + rec_offs_any_extern(clust_offsets); + + if (!extern_cols + || DBUG_IF("purge_no_blob_freeze")) { + /* Nothing to dereference: the stack of + versions being locked by mtr is enough. + purge_no_blob_freeze sends a version that does + have externally stored columns here as well, + which is how the dereference below behaved + before it was protected. */ + if (extern_cols) { + DEBUG_SYNC_C( + "row_undo_mod_sec_is_unsafe_row_build"); + } + row = row_build(ROW_COPY_POINTERS, clust_index, + prev_version, clust_offsets, + NULL, NULL, NULL, &ext, heap); + } else { + /* Freeze purge_sys.view across the + dereference and confirm that purge cannot see + the writer of version, the oldest one this + walk has applied. If it can, stop and report + the secondary index entry as still needed, + which is the conservative answer. Only the + dereference needs the freeze: the prefixes end + up copied into heap. */ + purge_sys_t::view_guard freeze{ + purge_sys_t::view_guard::VIEW}; + + /* row_undo_mod_sec_is_unsafe_purgeable forces + this branch, which is otherwise reached only + when the view advances between the walk + starting and the freeze. */ + if (freeze.view().changes_visible( + version_trx_id) + || DBUG_IF( + "row_undo_mod_sec_is_unsafe_purgeable")) { + break; + } + + DEBUG_SYNC_C( + "row_undo_mod_sec_is_unsafe_row_build"); + + row = row_build(ROW_COPY_POINTERS, clust_index, + prev_version, clust_offsets, + NULL, NULL, NULL, &ext, heap); + } if (dict_index_has_virtual(index)) { ut_ad(cur_vrow); diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 32b17ba1cc0..d71858ff9a8 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -253,13 +253,47 @@ not_locked: prev_trx_id = row_get_rec_trx_id(prev_version, clust_index, clust_offsets); - /* The stack of versions is locked by mtr. Thus, it - is safe to fetch the prefixes for externally stored - columns. */ + const bool extern_cols = rec_offs_any_extern(clust_offsets); - row = row_build(ROW_COPY_POINTERS, clust_index, prev_version, - clust_offsets, - NULL, NULL, NULL, &ext, heap); + if (!extern_cols || DBUG_IF("purge_no_blob_freeze")) { + /* Nothing to dereference: the stack of versions + being locked by mtr is enough. + purge_no_blob_freeze sends a version that does + have externally stored columns here as well, + which is how the dereference below behaved + before it was protected. */ + if (extern_cols) { + DEBUG_SYNC_C( + "row_vers_impl_x_locked_row_build"); + } + row = row_build(ROW_COPY_POINTERS, clust_index, + prev_version, clust_offsets, + NULL, NULL, NULL, &ext, heap); + } else { + /* Freeze purge_sys.view across the dereference and + confirm that purge has not seen trx, which wrote + every undo log record this walk applies (the loop + only continues while prev_trx_id is trx->id). If it + has, trx is committed and holds no implicit lock. + Only the dereference needs the freeze: the prefixes + end up copied into heap. */ + purge_sys_t::view_guard freeze{ + purge_sys_t::view_guard::VIEW}; + + /* row_vers_impl_x_locked_purgeable forces this + branch, which is otherwise reached only when trx + commits between the walk starting and the freeze. */ + if (freeze.view().changes_visible(trx->id) + || DBUG_IF("row_vers_impl_x_locked_purgeable")) { + goto not_locked; + } + + DEBUG_SYNC_C("row_vers_impl_x_locked_row_build"); + + row = row_build(ROW_COPY_POINTERS, clust_index, + prev_version, clust_offsets, + NULL, NULL, NULL, &ext, heap); + } if (dict_index_has_virtual(index)) { if (vrow) { diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 19cec8f269e..02706abb8e5 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -40,6 +40,7 @@ Created 3/26/1996 Heikki Tuuri #include "trx0rseg.h" #include "trx0trx.h" #include "dict0load.h" +#include "debug_sync.h" #include #include #include "log.h" @@ -1520,6 +1521,26 @@ ulint trx_purge(trx_t *trx, ulint n_tasks, ulint history_size) noexcept trx_purge_close_tables(thd, nullptr, true); } +#ifdef ENABLED_DEBUG_SYNC + /* Hold the batch open between its last purged record and the advance of + purge_sys.head and purge_sys.end_view, which is the window where a reader + that goes by end_view can still reach history this batch has removed. That + window is not otherwise accessible from a test, because it opens and closes + within this function. + + Only a batch that purged something opens it, and parking one that did not + would stop the batch a test is waiting for. Whoever sets purge_hold_cleanup + must release the batch: until then it keeps this thread, and the undo pages + the batch handled. */ + if (n_work) + DBUG_EXECUTE_IF("purge_hold_cleanup", + debug_sync_set_action + (current_thd, + STRING_WITH_LEN("now SIGNAL purge_batch_parked " + "WAIT_FOR purge_batch_resume")); + ); +#endif + purge_sys.batch_cleanup(head); MONITOR_INC_VALUE(MONITOR_PURGE_INVOKED, 1); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index cfff06ce9fc..09b5a058ff9 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -2246,16 +2246,19 @@ static dberr_t trx_undo_prev_version(const rec_t *rec, dict_index_t *index, cannot have purged the BLOBs referenced by that version yet). - This function does not fetch any BLOBs. The callers might, by - possibly invoking row_ext_create() via row_build(). However, - they should have all needed information in the *old_vers - returned by this function. This is because *old_vers is based - on the transaction undo log records. The function - trx_undo_page_fetch_ext() will write BLOB prefixes to the - transaction undo log that are at least as long as the longest - possible column prefix in a secondary index. Thus, secondary - index entries for *old_vers can be constructed without - dereferencing any BLOB pointers. */ + This function does not fetch any BLOBs, but the callers might, + by invoking row_ext_create() via row_build(). Both (a) and (b) + are statements about purge_sys.view, and they cover a reference + that the version merely inherited only if no transaction whose + undo log record was applied on the way down is visible to that + view, because one of those records disowned the reference. Such + a caller must therefore freeze purge_sys.view across the + dereference, by holding purge_sys.latch or by being a purge task, + and establish that condition under the freeze. Testing the + oldest writer applied suffices: a newer version can only have + been written once the older writer released the exclusive lock on + the record, so a view that sees a newer writer saw the older one + end before the view was created. */ ptr = trx_undo_rec_skip_row_ref(ptr, index); @@ -2271,18 +2274,12 @@ static dberr_t trx_undo_prev_version(const rec_t *rec, dict_index_t *index, records in secondary indexes, it normally covers some history that is already being purged. This is safe as long as the undo log records have not been freed yet. + Whether the externally stored columns of the version + built here may still be dereferenced is left to the + caller, the only one that knows whether it will + dereference any and can test that atomically with it. - However, BLOBs are only safe to access as long as the - purge_sys.view does not permit them to be freed. The - check.latch will freeze the purge_sys.view by blocking - purge_sys.clone_oldest_view() at the start of - trx_purge() or by blocking purge_sys.batch_cleanup() - at the end of trx_purge(). */ - if (check.is_extended() && purge_sys.is_purgeable(trx_id)) { - return DB_SUCCESS; - } - - /* We should confirm the existence of disowned external data, + We should confirm the existence of disowned external data, if the previous version record is delete marked. If the trx_id of the previous record is seen by purge view, we should treat it as missing history, because the disowned external data diff --git a/storage/maria/aria_pack.c b/storage/maria/aria_pack.c index 9518de85e61..2d07f4c8ca6 100644 --- a/storage/maria/aria_pack.c +++ b/storage/maria/aria_pack.c @@ -3027,6 +3027,8 @@ static void flush_bits(void) int bits; ulonglong bit_buffer; + if (file_buffer.bits == BITS_SAVED) /* Nothing to write */ + return; bits= file_buffer.bits & ~7; bit_buffer= file_buffer.bitbucket >> bits; bits= BITS_SAVED - bits; diff --git a/storage/myisam/myisampack.c b/storage/myisam/myisampack.c index e78e3c49129..b0f68f448a9 100644 --- a/storage/myisam/myisampack.c +++ b/storage/myisam/myisampack.c @@ -2939,6 +2939,9 @@ static void flush_bits(void) int bits; ulonglong bit_buffer; + if (file_buffer.bits == BITS_SAVED) /* Nothing to write */ + return; + bits= file_buffer.bits & ~7; if (bits != BITS_SAVED) { diff --git a/strings/json_normalize.c b/strings/json_normalize.c index 5243bd3c706..7557f18fd4d 100644 --- a/strings/json_normalize.c +++ b/strings/json_normalize.c @@ -829,7 +829,7 @@ json_norm_parse(struct json_norm_value *root, json_engine_t *je) json_norm_parse_end: dynstr_free(&key); - return err; + return err || je->s.error; } @@ -860,6 +860,10 @@ json_norm_build(json_engine_t *je, struct json_norm_value *root, if (err) return err; } + else + { + while(json_scan_next(je) == 0) /*no-op*/; + } return err; } @@ -908,14 +912,9 @@ json_normalize_engine(json_engine_t *je, DYNAMIC_STRING *result, in_size= strlen(s_utf8); } - - if (!json_valid_engine(je, in, in_size, &my_charset_utf8mb4_bin)) - { - err= 1; - goto json_normalize_end; - } - err= json_norm_build(je, &root, in, in_size, &my_charset_utf8mb4_bin); + if (root.type == JSON_VALUE_UNINITIALIZED || je->s.error) + err= 1; if (err) goto json_normalize_end;