From 08c05b5f34f7b301ba669a88df3b038f0c34d379 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 29 Jan 2019 14:18:35 +0200 Subject: [PATCH 01/44] MDEV-15744: Assertion `derived->table' failed in mysql_derived_merge_for_insert For singe-table views, we need to find the bottom most base table in the embedded views and then update that table --- mysql-test/r/loaddata.result | 24 ++++++++++++++++++++++++ mysql-test/t/loaddata.test | 14 ++++++++++++++ sql/sql_load.cc | 5 +++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index 0421c591aff..e09d1bee5f4 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -550,3 +550,27 @@ SELECT HEX(a) FROM t1; HEX(a) C3A4 DROP TABLE t1; +# +# MDEV-15744: Assertion `derived->table' failed in mysql_derived_merge_for_insert +# +create table t1 (a int, b int); +CREATE OR REPLACE VIEW t2 AS SELECT * FROM t1; +CREATE VIEW v2 AS SELECT * FROM t2; +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v2 +FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; +select * from v2; +a b +2 2 +3 3 +4 4 +5 5 +6 6 +select * from t2; +a b +2 2 +3 3 +4 4 +5 5 +6 6 +DROP VIEW IF EXISTS v2,t2; +DROP TABLE IF EXISTS t1; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 8f8ff3520a2..50d5615e58c 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -676,3 +676,17 @@ CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; SELECT HEX(a) FROM t1; DROP TABLE t1; + +--echo # +--echo # MDEV-15744: Assertion `derived->table' failed in mysql_derived_merge_for_insert +--echo # + +create table t1 (a int, b int); +CREATE OR REPLACE VIEW t2 AS SELECT * FROM t1; +CREATE VIEW v2 AS SELECT * FROM t2; +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v2 + FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; +select * from v2; +select * from t2; +DROP VIEW IF EXISTS v2,t2; +DROP TABLE IF EXISTS t1; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 5029efa7d68..27c0938c9a0 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -233,8 +233,9 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, if (open_and_lock_tables(thd, table_list, TRUE, 0)) DBUG_RETURN(TRUE); - if (mysql_handle_single_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT) || - mysql_handle_single_derived(thd->lex, table_list, DT_PREPARE)) + if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT)) + DBUG_RETURN(TRUE); + if (mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE)) DBUG_RETURN(TRUE); if (setup_tables_and_check_access(thd, &thd->lex->select_lex.context, &thd->lex->select_lex.top_join_list, From 6092093cb99a71b0aec02b785087c0df307b1a66 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 30 Jan 2019 19:35:40 +0530 Subject: [PATCH 02/44] MDEV-15950: LOAD DATA INTO compex_view crashed For multi-table views with LOAD, updates are not allowed, so we should just throw an error. --- mysql-test/r/loaddata.result | 26 ++++++++++++++++++++++++++ mysql-test/t/loaddata.test | 31 +++++++++++++++++++++++++++++++ sql/sql_load.cc | 5 +++++ 3 files changed, 62 insertions(+) diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index e09d1bee5f4..73ebbd5590c 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -574,3 +574,29 @@ a b 6 6 DROP VIEW IF EXISTS v2,t2; DROP TABLE IF EXISTS t1; +# +# MDEV-15950: LOAD DATA INTO compex_view crashed +# +create table t1 (a int, b int); +create table t0 (x int, y int); +CREATE OR REPLACE VIEW v1 AS SELECT * FROM t1,t0; +CREATE VIEW v2 AS SELECT * FROM v1; +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v1 +FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; +ERROR HY000: Incorrect usage of Multi-table VIEW and LOAD +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v2 +FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; +ERROR HY000: Incorrect usage of Multi-table VIEW and LOAD +DROP VIEW IF EXISTS v2,v1; +DROP TABLE IF EXISTS t1,t0; +CREATE TABLE t1 (a INT, b INT, PRIMARY KEY (a), UNIQUE(b)); +INSERT INTO t1 VALUES (1,1); +CREATE TABLE t2 (c INT); +CREATE VIEW v AS SELECT t1.* FROM t1 JOIN t2; +SELECT a, b FROM t1 INTO OUTFILE '15645.data'; +LOAD DATA INFILE '15645.data' IGNORE INTO TABLE v (a,b); +ERROR HY000: Incorrect usage of Multi-table VIEW and LOAD +LOAD DATA INFILE '15645.data' REPLACE INTO TABLE v (a,b); +ERROR HY000: Incorrect usage of Multi-table VIEW and LOAD +drop table t1,t2; +drop view v; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 50d5615e58c..de9de4305a3 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -690,3 +690,34 @@ select * from v2; select * from t2; DROP VIEW IF EXISTS v2,t2; DROP TABLE IF EXISTS t1; + +--echo # +--echo # MDEV-15950: LOAD DATA INTO compex_view crashed +--echo # +create table t1 (a int, b int); +create table t0 (x int, y int); +CREATE OR REPLACE VIEW v1 AS SELECT * FROM t1,t0; +CREATE VIEW v2 AS SELECT * FROM v1; + +--error ER_WRONG_USAGE +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v1 +FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; + +--error ER_WRONG_USAGE +LOAD DATA INFILE '../../std_data/loaddata7.dat' INTO TABLE v2 +FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n'; + +DROP VIEW IF EXISTS v2,v1; +DROP TABLE IF EXISTS t1,t0; + +CREATE TABLE t1 (a INT, b INT, PRIMARY KEY (a), UNIQUE(b)); +INSERT INTO t1 VALUES (1,1); +CREATE TABLE t2 (c INT); +CREATE VIEW v AS SELECT t1.* FROM t1 JOIN t2; +SELECT a, b FROM t1 INTO OUTFILE '15645.data'; +--error ER_WRONG_USAGE +LOAD DATA INFILE '15645.data' IGNORE INTO TABLE v (a,b); +--error ER_WRONG_USAGE +LOAD DATA INFILE '15645.data' REPLACE INTO TABLE v (a,b); +drop table t1,t2; +drop view v; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 27c0938c9a0..113cbe1dac0 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -251,6 +251,11 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, my_error(ER_NON_UPDATABLE_TABLE, MYF(0), table_list->alias, "LOAD"); DBUG_RETURN(TRUE); } + if (table_list->is_multitable()) + { + my_error(ER_WRONG_USAGE, MYF(0), "Multi-table VIEW", "LOAD"); + DBUG_RETURN(TRUE); + } if (table_list->prepare_where(thd, 0, TRUE) || table_list->prepare_check_option(thd)) { From 9034e5e18e756eb8d670420d1f180b8eaa4cc27b Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 30 Jan 2019 10:12:21 -0500 Subject: [PATCH 03/44] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 34a2418fde1..1fb4ae866ce 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=63 +MYSQL_VERSION_PATCH=64 MYSQL_VERSION_EXTRA= From c9b9d9f5152b2ec16e88c0b235d50420c52b41de Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 7 Feb 2019 16:46:39 +0100 Subject: [PATCH 04/44] MDEV-18506 MSI can't be built if MFC package is not installed with Visual Studio candle.exe's preprocessor flags (-dHaveUpgradeWizard=0 -DHaveInnodb=1) were not passed correctly to EXECUTE_PROCESS Fix is to make a list out of the EXTRA_WIX_PREPROCESSOR_FLAGS string, and use the preprocessor flags list in EXECUTE_PROCESS. --- win/packaging/create_msi.cmake.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake.in index fceea041676..0dfac8db6fd 100644 --- a/win/packaging/create_msi.cmake.in +++ b/win/packaging/create_msi.cmake.in @@ -401,9 +401,10 @@ IF("$ENV{EXTRA_LIGHT_ARGS}") ENDIF() FILE(REMOVE mysql_server.wixobj extra.wixobj) +STRING(REPLACE " " ";" EXTRA_WIX_PREPROCESSOR_FLAGS_LIST ${EXTRA_WIX_PREPROCESSOR_FLAGS}) EXECUTE_PROCESS( COMMAND ${CANDLE_EXECUTABLE} - ${EXTRA_WIX_PREPROCESSOR_FLAGS} + ${EXTRA_WIX_PREPROCESSOR_FLAGS_LIST} ${CANDLE_ARCH} -ext WixUtilExtension -ext WixFirewallExtension @@ -413,7 +414,7 @@ EXECUTE_PROCESS( EXECUTE_PROCESS( COMMAND ${CANDLE_EXECUTABLE} ${CANDLE_ARCH} - ${EXTRA_WIX_PREPROCESSOR_FLAGS} + ${EXTRA_WIX_PREPROCESSOR_FLAGS_LIST} -ext WixUtilExtension -ext WixFirewallExtension ${CMAKE_CURRENT_BINARY_DIR}/extra.wxs From 3262967008bf2c2978bbf97960a3ea168fec9c26 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 14 Feb 2019 11:11:16 +0100 Subject: [PATCH 05/44] MDEV-18575 Cleanup : remove innodb-encrypt-log parameter from mariabackup --- extra/mariabackup/xtrabackup.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 2d28a6caa84..d29c570ab37 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -566,7 +566,6 @@ enum options_xtrabackup OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_PLUGIN_LOAD, - OPT_INNODB_ENCRYPT_LOG, OPT_CLOSE_FILES, OPT_CORE_FILE, @@ -1159,10 +1158,6 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, &xb_plugin_load, &xb_plugin_load, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", - &srv_encrypt_log, &srv_encrypt_log, - 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, From a2f82b649d39ec8e5e321430adf1d1ed13f86174 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 20 Feb 2019 16:23:10 +0100 Subject: [PATCH 06/44] MDEV-17942 Assertion `found' failed in remove_ptr_from_dynarray after failed CREATE OR REPLACE Failed CREATE OR REPLACE for existing user removes that user from acl_users array. Thus dependend structures (roles, check_host) must be rebuilt. --- mysql-test/r/create_drop_role.result | 9 +++++++++ mysql-test/t/create_drop_role.test | 11 +++++++++++ sql/sql_acl.cc | 11 +++++++++++ 3 files changed, 31 insertions(+) diff --git a/mysql-test/r/create_drop_role.result b/mysql-test/r/create_drop_role.result index 6e05900264d..d77fc0afc3a 100644 --- a/mysql-test/r/create_drop_role.result +++ b/mysql-test/r/create_drop_role.result @@ -73,3 +73,12 @@ Note 1976 Can't drop role 'role_1'; it doesn't exist DROP ROLE role_1; ERROR HY000: Operation DROP ROLE failed for 'role_1' DROP USER u1@localhost; +CREATE ROLE r; +GRANT SHOW DATABASES ON *.* TO r; +CREATE USER foo; +CREATE USER bar; +GRANT r TO foo; +CREATE OR REPLACE USER foo IDENTIFIED WITH non_existing_plugin; +ERROR HY000: Plugin 'non_existing_plugin' is not loaded +DROP ROLE r; +DROP USER bar; diff --git a/mysql-test/t/create_drop_role.test b/mysql-test/t/create_drop_role.test index be33083e6c4..11bc832c6e9 100644 --- a/mysql-test/t/create_drop_role.test +++ b/mysql-test/t/create_drop_role.test @@ -54,3 +54,14 @@ DROP ROLE IF EXISTS role_1; DROP ROLE role_1; DROP USER u1@localhost; + +# MDEV-17942 +CREATE ROLE r; +GRANT SHOW DATABASES ON *.* TO r; +CREATE USER foo; +CREATE USER bar; +GRANT r TO foo; +--error ER_PLUGIN_IS_NOT_LOADED +CREATE OR REPLACE USER foo IDENTIFIED WITH non_existing_plugin; +DROP ROLE r; +DROP USER bar; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 9d6e6c51fb6..a7d1f6a1b38 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -9439,6 +9439,7 @@ bool mysql_create_user(THD *thd, List &list, bool handle_as_role) List_iterator user_list(list); TABLE_LIST tables[TABLES_MAX]; bool binlog= false; + bool some_users_dropped= false; DBUG_ENTER("mysql_create_user"); DBUG_PRINT("entry", ("Handle as %s", handle_as_role ? "role" : "user")); @@ -9504,6 +9505,8 @@ bool mysql_create_user(THD *thd, List &list, bool handle_as_role) result= true; continue; } + else + some_users_dropped= true; // Proceed with the creation } else if (thd->lex->create_info.if_not_exists()) @@ -9573,9 +9576,17 @@ bool mysql_create_user(THD *thd, List &list, bool handle_as_role) mysql_mutex_unlock(&acl_cache->lock); if (result) + { + if (some_users_dropped && !handle_as_role) + { + /* Rebuild in-memory structs, since 'acl_users' has been modified */ + rebuild_check_host(); + rebuild_role_grants(); + } my_error(ER_CANNOT_USER, MYF(0), (handle_as_role) ? "CREATE ROLE" : "CREATE USER", wrong_users.c_ptr_safe()); + } if (binlog) result |= write_bin_log(thd, FALSE, thd->query(), thd->query_length()); From 39a2984dc00925c8e08917c9313eaa03b82011f4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 20 Feb 2019 17:22:44 +0100 Subject: [PATCH 07/44] Revert "MDEV-18575 Cleanup : remove innodb-encrypt-log parameter from mariabackup" This reverts commit 3262967008bf2c2978bbf97960a3ea168fec9c26. It was checked in by mistake --- extra/mariabackup/xtrabackup.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d29c570ab37..2d28a6caa84 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -566,6 +566,7 @@ enum options_xtrabackup OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_PLUGIN_LOAD, + OPT_INNODB_ENCRYPT_LOG, OPT_CLOSE_FILES, OPT_CORE_FILE, @@ -1158,6 +1159,10 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, &xb_plugin_load, &xb_plugin_load, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", + &srv_encrypt_log, &srv_encrypt_log, + 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, From d9f7b6be5aa8d0eea18d1f122338b6058c03fdc3 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 20 Feb 2019 22:35:21 +0100 Subject: [PATCH 08/44] MDEV-17942 fixup : protect rebuild_check_host() / rebuild_role_grants() with acl_cache->lock mutex --- sql/sql_acl.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index a7d1f6a1b38..6052284428e 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -9573,16 +9573,17 @@ bool mysql_create_user(THD *thd, List &list, bool handle_as_role) } } + if (result && some_users_dropped && !handle_as_role) + { + /* Rebuild in-memory structs, since 'acl_users' has been modified */ + rebuild_check_host(); + rebuild_role_grants(); + } + mysql_mutex_unlock(&acl_cache->lock); if (result) { - if (some_users_dropped && !handle_as_role) - { - /* Rebuild in-memory structs, since 'acl_users' has been modified */ - rebuild_check_host(); - rebuild_role_grants(); - } my_error(ER_CANNOT_USER, MYF(0), (handle_as_role) ? "CREATE ROLE" : "CREATE USER", wrong_users.c_ptr_safe()); From 945c748adc5475e59e5d224f1543756759e4e668 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 21 Feb 2019 00:06:08 +0100 Subject: [PATCH 09/44] MDEV-18669 mariabackup writes timestamp in version line Use fprintf(stderr) instead of msg() to print the version line --- extra/mariabackup/xtrabackup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index acd11135289..96537e3de42 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1591,7 +1591,7 @@ static const char *xb_server_default_groups[]={ static void print_version(void) { - msg("%s based on MariaDB server %s %s (%s)", + fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n", my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); } From b88a803459c479f41bbff0a2bfe4c8d90cd1c446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 21 Feb 2019 09:19:18 +0200 Subject: [PATCH 10/44] MDEV-17428: Update wsrep_max_ws_rows and wsrep_max_ws_size values in wsrep.cnf.sh Since MariaDB 10.1.17 the new default values for wsrep_max_ws_rows and wsrep_max_ws_size were set: wsrep_max_ws_rows Default Value: 0 (>= MariaDB Galera 10.0.27, MariaDB 10.1.17) 131072 (<= MariaDB Galera 10.0.26, MariaDB 10.1.16) wsrep_max_ws_size Default Value: 2147483647 (2GB, >= MariaDB Galera 10.0.27, MariaDB 10.1.17) 1073741824 (1GB, <= MariaDB Galera 10.0.26, MariaDB 10.1.16) --- support-files/wsrep.cnf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh index 51ce3dca2dd..7242fff9f7a 100644 --- a/support-files/wsrep.cnf.sh +++ b/support-files/wsrep.cnf.sh @@ -67,10 +67,10 @@ wsrep_slave_threads=1 wsrep_certify_nonPK=1 # Maximum number of rows in write set -wsrep_max_ws_rows=131072 +wsrep_max_ws_rows=0 # Maximum size of write set -wsrep_max_ws_size=1073741824 +wsrep_max_ws_size=2147483647 # to enable debug level logging, set this to 1 wsrep_debug=0 From 28cb041754f9f77e915ecd9c5276e02a1ab4cec5 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 20 Feb 2019 14:56:02 +0300 Subject: [PATCH 11/44] MDEV-18662 ib_wqueue_t has a data race ib_wqueue_is_empty(): protect ib_list_is_empty() call Closes #1202 --- storage/innobase/include/ut0wqueue.h | 15 +++++---------- storage/innobase/ut/ut0wqueue.cc | 19 +++++++++---------- storage/xtradb/include/ut0wqueue.h | 15 +++++---------- storage/xtradb/ut/ut0wqueue.cc | 19 +++++++++---------- 4 files changed, 28 insertions(+), 40 deletions(-) diff --git a/storage/innobase/include/ut0wqueue.h b/storage/innobase/include/ut0wqueue.h index 47930145426..dcd3feabe72 100644 --- a/storage/innobase/include/ut0wqueue.h +++ b/storage/innobase/include/ut0wqueue.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2009, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -67,15 +67,10 @@ ib_wqueue_add( mem_heap_t* heap); /*!< in: memory heap to use for allocating the list node */ -/******************************************************************** -Check if queue is empty. */ - -ibool -ib_wqueue_is_empty( -/*===============*/ - /* out: TRUE if queue empty - else FALSE */ - const ib_wqueue_t* wq); /* in: work queue */ +/** Check if queue is empty. +@param wq wait queue +@return whether the queue is empty */ +bool ib_wqueue_is_empty(ib_wqueue_t* wq); /****************************************************************//** Wait for a work item to appear in the queue. diff --git a/storage/innobase/ut/ut0wqueue.cc b/storage/innobase/ut/ut0wqueue.cc index 1607e535a94..fce39bd817e 100644 --- a/storage/innobase/ut/ut0wqueue.cc +++ b/storage/innobase/ut/ut0wqueue.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -193,17 +194,15 @@ ib_wqueue_nowait( return (node ? node->data : NULL); } -/******************************************************************** -Check if queue is empty. */ - -ibool -ib_wqueue_is_empty( -/*===============*/ - /* out: TRUE if queue empty - else FALSE */ - const ib_wqueue_t* wq) /* in: work queue */ +/** Check if queue is empty. +@param wq wait queue +@return whether the queue is empty */ +bool ib_wqueue_is_empty(ib_wqueue_t* wq) { - return(ib_list_is_empty(wq->items)); + mutex_enter(&wq->mutex); + bool is_empty = ib_list_is_empty(wq->items); + mutex_exit(&wq->mutex); + return is_empty; } /******************************************************************** diff --git a/storage/xtradb/include/ut0wqueue.h b/storage/xtradb/include/ut0wqueue.h index d69363afe7b..e0f8f76d5f1 100644 --- a/storage/xtradb/include/ut0wqueue.h +++ b/storage/xtradb/include/ut0wqueue.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2009, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -67,15 +67,10 @@ ib_wqueue_add( mem_heap_t* heap); /*!< in: memory heap to use for allocating the list node */ -/******************************************************************** -Check if queue is empty. */ - -ibool -ib_wqueue_is_empty( -/*===============*/ - /* out: TRUE if queue empty - else FALSE */ - const ib_wqueue_t* wq); /* in: work queue */ +/** Check if queue is empty. +@param wq wait queue +@return whether the queue is empty */ +bool ib_wqueue_is_empty(ib_wqueue_t* wq); /****************************************************************//** Wait for a work item to appear in the queue. diff --git a/storage/xtradb/ut/ut0wqueue.cc b/storage/xtradb/ut/ut0wqueue.cc index 1607e535a94..fce39bd817e 100644 --- a/storage/xtradb/ut/ut0wqueue.cc +++ b/storage/xtradb/ut/ut0wqueue.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -193,17 +194,15 @@ ib_wqueue_nowait( return (node ? node->data : NULL); } -/******************************************************************** -Check if queue is empty. */ - -ibool -ib_wqueue_is_empty( -/*===============*/ - /* out: TRUE if queue empty - else FALSE */ - const ib_wqueue_t* wq) /* in: work queue */ +/** Check if queue is empty. +@param wq wait queue +@return whether the queue is empty */ +bool ib_wqueue_is_empty(ib_wqueue_t* wq) { - return(ib_list_is_empty(wq->items)); + mutex_enter(&wq->mutex); + bool is_empty = ib_list_is_empty(wq->items); + mutex_exit(&wq->mutex); + return is_empty; } /******************************************************************** From 4946eb7b54323f98fa2e399e981467733c3b0914 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Thu, 21 Feb 2019 10:50:54 -0500 Subject: [PATCH 12/44] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0e5b7b386b1..322038a4bdd 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=3 -MYSQL_VERSION_PATCH=13 +MYSQL_VERSION_PATCH=14 SERVER_MATURITY=stable From 2c8d9a4e5974304f82505832ae05360401a417cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 22 Feb 2019 16:21:03 +0200 Subject: [PATCH 13/44] MDEV-10813: Update buf_page_t::buf_fix_count outside mutex Since MySQL 5.6.16 (and MariaDB Server 10.0.11), changes of buf_page_t::buf_fix_count are atomic memory operations if PAGE_ATOMIC_REF_COUNT is defined. Since MySQL 5.7 (and MariaDB Server 10.2.2), the field is always updated by atomic memory operations. In a few occurrences, updates of the counter were unnecessarily surrounded by an acquisition and release of the block mutex (buf_block_t::mutex or buf_pool_t::zip_mutex). Remove these unnecessary mutex operations. --- storage/innobase/btr/btr0cur.cc | 7 ------- storage/innobase/buf/buf0buf.cc | 12 ------------ storage/innobase/fsp/fsp0fsp.cc | 3 --- 3 files changed, 22 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 9cf42a93800..f1ecaca3b54 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -463,11 +463,7 @@ btr_cur_optimistic_latch_leaves( file, line, mtr)) { if (btr_page_get_prev(buf_block_get_frame(block), mtr) == left_page_no) { - /* adjust buf_fix_count */ - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - *latch_mode = mode; return(true); } else { @@ -483,10 +479,7 @@ btr_cur_optimistic_latch_leaves( } unpin_failed: /* unpin the block */ - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - return(false); default: diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 6f00ba7d6d4..e5ba697a77f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4915,10 +4915,7 @@ buf_page_optimistic_get( } if (!success) { - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - return(FALSE); } @@ -4932,10 +4929,7 @@ buf_page_optimistic_get( rw_lock_x_unlock(&block->lock); } - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - return(FALSE); } @@ -5040,10 +5034,7 @@ buf_page_get_known_nowait( } if (!success) { - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - return(FALSE); } @@ -5137,10 +5128,7 @@ buf_page_try_get_func( } if (!success) { - buf_page_mutex_enter(block); buf_block_buf_fix_dec(block); - buf_page_mutex_exit(block); - return(NULL); } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 11c5c65d861..2e128594a4a 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1374,11 +1374,8 @@ fsp_page_create( } else { rw_lock_sx_lock(&block->lock); } - mutex_enter(&block->mutex); buf_block_buf_fix_inc(block, __FILE__, __LINE__); - - mutex_exit(&block->mutex); mtr_memo_push(init_mtr, block, rw_latch == RW_X_LATCH ? MTR_MEMO_PAGE_X_FIX : MTR_MEMO_PAGE_SX_FIX); From 09bd2138522787a4e0b015695c462903f4a9e728 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 22 Feb 2019 21:38:55 -0800 Subject: [PATCH 14/44] MDEV-18700 EXPLAIN EXTENDED shows a wrong operation for query with UNION ALL after INTERSECT EXPLAIN EXTENDED erroneously showed UNION instead of UNION ALL in the warning if UNION ALL followed INTERSECT or EXCEPT operations. The bug was in the function st_select_lex_unit::print() that printed the text of the query used in the warning. --- mysql-test/main/union.result | 37 ++++++++++++++++++++++++++++++++++++ mysql-test/main/union.test | 30 +++++++++++++++++++++++++++++ sql/sql_lex.cc | 4 ++-- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/union.result b/mysql-test/main/union.result index da99d65dec3..ef767b1d5af 100644 --- a/mysql-test/main/union.result +++ b/mysql-test/main/union.result @@ -2568,5 +2568,42 @@ c1 -10 drop table t1,t2; # +# MDEV-18700: EXPLAIN EXTENDED for query with UNION ALL +# after INTERSECT/EXCEPT operations +# +create table t1 (a int); +insert into t1 values (3), (1), (7), (3), (2), (7), (4); +create table t2 (a int); +insert into t2 values (4), (5), (9), (1), (8), (9); +create table t3 (a int); +insert into t3 values (8), (1), (8), (2), (3), (7), (2); +explain extended +select * from t2 where a < 5 +intersect +select * from t3 where a < 5 +union all +select * from t1 where a > 4; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 6 100.00 Using where +2 INTERSECT t3 ALL NULL NULL NULL NULL 7 100.00 Using where +3 UNION t1 ALL NULL NULL NULL NULL 7 100.00 Using where +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL NULL +Warnings: +Note 1003 /* select#1 */ select `test`.`t2`.`a` AS `a` from `test`.`t2` where `test`.`t2`.`a` < 5 intersect /* select#2 */ select `test`.`t3`.`a` AS `a` from `test`.`t3` where `test`.`t3`.`a` < 5 union all /* select#3 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 4 +explain extended +select * from t2 where a < 5 +except +select * from t3 where a < 5 +union all +select * from t1 where a > 4; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 6 100.00 Using where +2 EXCEPT t3 ALL NULL NULL NULL NULL 7 100.00 Using where +3 UNION t1 ALL NULL NULL NULL NULL 7 100.00 Using where +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL NULL +Warnings: +Note 1003 /* select#1 */ select `test`.`t2`.`a` AS `a` from `test`.`t2` where `test`.`t2`.`a` < 5 except /* select#2 */ select `test`.`t3`.`a` AS `a` from `test`.`t3` where `test`.`t3`.`a` < 5 union all /* select#3 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 4 +drop table t1,t2,t3; +# # End of 10.3 tests # diff --git a/mysql-test/main/union.test b/mysql-test/main/union.test index e7543ba4a81..9d9194ebab6 100644 --- a/mysql-test/main/union.test +++ b/mysql-test/main/union.test @@ -1811,6 +1811,36 @@ SELECT c1 FROM t1 UNION SELECT - @a FROM t2; drop table t1,t2; +--echo # +--echo # MDEV-18700: EXPLAIN EXTENDED for query with UNION ALL +--echo # after INTERSECT/EXCEPT operations +--echo # + +create table t1 (a int); +insert into t1 values (3), (1), (7), (3), (2), (7), (4); + +create table t2 (a int); +insert into t2 values (4), (5), (9), (1), (8), (9); + +create table t3 (a int); +insert into t3 values (8), (1), (8), (2), (3), (7), (2); + +explain extended +select * from t2 where a < 5 +intersect +select * from t3 where a < 5 +union all +select * from t1 where a > 4; + +explain extended +select * from t2 where a < 5 +except +select * from t3 where a < 5 +union all +select * from t1 where a > 4; + +drop table t1,t2,t3; + --echo # --echo # End of 10.3 tests --echo # diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 430bd2b65c3..4282a8fb8ae 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -2897,8 +2897,6 @@ void st_select_lex_unit::print(String *str, enum_query_type query_type) str->append(STRING_WITH_LEN(" union ")); if (union_all) str->append(STRING_WITH_LEN("all ")); - else if (union_distinct == sl) - union_all= TRUE; break; case INTERSECT_TYPE: str->append(STRING_WITH_LEN(" intersect ")); @@ -2907,6 +2905,8 @@ void st_select_lex_unit::print(String *str, enum_query_type query_type) str->append(STRING_WITH_LEN(" except ")); break; } + if (sl == union_distinct) + union_all= TRUE; } if (sl->braces) str->append('('); From 15a20367fbe0f0c726caba92d541ab7f04b830b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 23 Feb 2019 10:31:07 +0200 Subject: [PATCH 15/44] buf_page_get_zip(): Deduplicate some code --- storage/innobase/buf/buf0buf.cc | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index e5ba697a77f..df607a6fc7a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -3782,18 +3782,10 @@ err_exit: ut_ad(!buf_pool_watch_is_sentinel(buf_pool, bpage)); switch (buf_page_get_state(bpage)) { - case BUF_BLOCK_POOL_WATCH: - case BUF_BLOCK_NOT_USED: - case BUF_BLOCK_READY_FOR_USE: - case BUF_BLOCK_MEMORY: - case BUF_BLOCK_REMOVE_HASH: - ut_error; - case BUF_BLOCK_ZIP_PAGE: case BUF_BLOCK_ZIP_DIRTY: buf_block_fix(bpage); block_mutex = &buf_pool->zip_mutex; - mutex_enter(block_mutex); goto got_block; case BUF_BLOCK_FILE_PAGE: /* Discard the uncompressed page frame if possible. */ @@ -3808,16 +3800,16 @@ err_exit: __FILE__, __LINE__); block_mutex = &((buf_block_t*) bpage)->mutex; - - mutex_enter(block_mutex); - goto got_block; + default: + break; } ut_error; goto err_exit; got_block: + mutex_enter(block_mutex); must_read = buf_page_get_io_fix(bpage) == BUF_IO_READ; rw_lock_s_unlock(hash_lock); From 0ef8848526c117ed8731e42edbb3f28652b168f2 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Dec 2017 13:22:45 +0400 Subject: [PATCH 16/44] Backporting MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode This is a part of "MDEV-18045 Backporting the MDEV-15497 changes to 10.2 branch" --- mysql-test/r/loadxml.result | 20 ++++++++ mysql-test/std_data/loaddata/mdev14628a.xml | 4 ++ mysql-test/std_data/loaddata/mdev14628b.xml | 3 ++ mysql-test/t/loadxml.test | 19 ++++++++ sql/field.cc | 51 +++++++++++++++++++++ sql/field.h | 4 ++ sql/sql_load.cc | 47 ++++--------------- 7 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 mysql-test/std_data/loaddata/mdev14628a.xml create mode 100644 mysql-test/std_data/loaddata/mdev14628b.xml diff --git a/mysql-test/r/loadxml.result b/mysql-test/r/loadxml.result index 1d7af4f8b38..b0fb867a676 100644 --- a/mysql-test/r/loadxml.result +++ b/mysql-test/r/loadxml.result @@ -134,3 +134,23 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2); ERROR HY000: Column 'c2' is not updatable DROP VIEW v1; DROP TABLE t1; +# +# MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode +# +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT); +LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1 ORDER BY b; +a b +1 bbb1 +2 bbb2 +DROP TABLE t1; +SET sql_mode=DEFAULT; +SET sql_mode=''; +CREATE TABLE t1 (id INT, g GEOMETRY NOT NULL); +LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +ERROR 22004: Column set to default value; NULL supplied to NOT NULL column 'g' at row 1 +SELECT * FROM t1; +id g +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/mysql-test/std_data/loaddata/mdev14628a.xml b/mysql-test/std_data/loaddata/mdev14628a.xml new file mode 100644 index 00000000000..34ee7336a5a --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev14628a.xml @@ -0,0 +1,4 @@ + + + + diff --git a/mysql-test/std_data/loaddata/mdev14628b.xml b/mysql-test/std_data/loaddata/mdev14628b.xml new file mode 100644 index 00000000000..2ea02d2a35f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev14628b.xml @@ -0,0 +1,3 @@ + + + diff --git a/mysql-test/t/loadxml.test b/mysql-test/t/loadxml.test index 0bd97a81649..623deea6ec6 100644 --- a/mysql-test/t/loadxml.test +++ b/mysql-test/t/loadxml.test @@ -143,3 +143,22 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c1); LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2); DROP VIEW v1; DROP TABLE t1; + +--echo # +--echo # MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode +--echo # + +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT); +LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1 ORDER BY b; +DROP TABLE t1; +SET sql_mode=DEFAULT; + +SET sql_mode=''; +CREATE TABLE t1 (id INT, g GEOMETRY NOT NULL); +--error ER_WARN_NULL_TO_NOTNULL +LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1; +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/sql/field.cc b/sql/field.cc index 855cc5e0b79..cfd51da1487 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1361,6 +1361,20 @@ warn: } +bool Field::load_data_set_null(THD *thd) +{ + reset(); + set_null(); + if (!maybe_null()) + { + if (this != table->next_number_field) + set_warning(Sql_condition::WARN_LEVEL_WARN, ER_WARN_NULL_TO_NOTNULL, 1); + } + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + /** Numeric fields base class constructor. */ @@ -5348,6 +5362,27 @@ int Field_timestamp::set_time() return 0; } + +bool Field_timestamp::load_data_set_null(THD *thd) +{ + if (!maybe_null()) + { + /* + Timestamp fields that are NOT NULL are autoupdated if there is no + corresponding value in the data file. + */ + set_time(); + } + else + { + reset(); + set_null(); + } + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + #ifdef NOT_USED static void store_native(ulonglong num, uchar *to, uint bytes) { @@ -8699,6 +8734,22 @@ bool Field_geom::can_optimize_range(const Item_bool_func *cond, return item->cmp_type() == STRING_RESULT; } + +bool Field_geom::load_data_set_null(THD *thd) +{ + Field_blob::reset(); + if (!maybe_null()) + { + my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field_name, + thd->get_stmt_da()->current_row_for_warning()); + return true; + } + set_null(); + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + #endif /*HAVE_SPATIAL*/ /**************************************************************************** diff --git a/sql/field.h b/sql/field.h index 55c3ed4c4bd..728b1f68880 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1143,6 +1143,8 @@ public: { if (null_ptr) null_ptr[row_offset]&= (uchar) ~null_bit; } inline bool maybe_null(void) const { return null_ptr != 0 || table->maybe_null; } + // Set to NULL on LOAD DATA or LOAD XML + virtual bool load_data_set_null(THD *thd); /* @return true if this field is NULL-able (even if temporarily) */ inline bool real_maybe_null(void) const { return null_ptr != 0; } @@ -2482,6 +2484,7 @@ public: { return get_equal_const_item_datetime(thd, ctx, const_item); } + bool load_data_set_null(THD *thd); uint size_of() const { return sizeof(*this); } }; @@ -3502,6 +3505,7 @@ public: but the underlying blob must still be reset. */ int reset(void) { return Field_blob::reset() || !maybe_null(); } + bool load_data_set_null(THD *thd); geometry_type get_geometry_type() { return geom_type; }; static geometry_type geometry_type_merge(geometry_type, geometry_type); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 70a6787d63b..da3aafa58e1 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -1053,7 +1053,6 @@ continue_loop:; } - static int read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, List &fields_vars, List &set_fields, @@ -1132,28 +1131,9 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } else { - Field *field= real_item->field; - if (field->reset()) - { - my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field->field_name, - thd->get_stmt_da()->current_row_for_warning()); + DBUG_ASSERT(real_item->field->table == table); + if (real_item->field->load_data_set_null(thd)) DBUG_RETURN(1); - } - field->set_null(); - if (!field->maybe_null()) - { - /* - Timestamp fields that are NOT NULL are autoupdated if there is no - corresponding value in the data file. - */ - if (field->type() == MYSQL_TYPE_TIMESTAMP) - field->set_time(); - else if (field != table->next_number_field) - field->set_warning(Sql_condition::WARN_LEVEL_WARN, - ER_WARN_NULL_TO_NOTNULL, 1); - } - /* Do not auto-update this field. */ - field->set_has_explicit_value(); } continue; @@ -1301,6 +1281,7 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, for ( ; ; it.rewind()) { + bool err; if (thd->killed) { thd->send_kill_message(); @@ -1352,21 +1333,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } else { - Field *field= real_item->field; - field->reset(); - field->set_null(); - if (field == table->next_number_field) - table->auto_increment_field_not_null= TRUE; - if (!field->maybe_null()) - { - if (field->type() == FIELD_TYPE_TIMESTAMP) - field->set_time(); - else if (field != table->next_number_field) - field->set_warning(Sql_condition::WARN_LEVEL_WARN, - ER_WARN_NULL_TO_NOTNULL, 1); - } - /* Do not auto-update this field. */ - field->set_has_explicit_value(); + DBUG_ASSERT(real_item->field->table == table); + if (real_item->field->load_data_set_null(thd)) + DBUG_RETURN(1); } continue; } @@ -1450,7 +1419,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } WSREP_LOAD_DATA_SPLIT(thd, table, info); - if (write_record(thd, table, &info)) + err= write_record(thd, table, &info); + table->auto_increment_field_not_null= false; + if (err) DBUG_RETURN(1); /* From 8036ad541e9da4073a6136052e41c22c758b770e Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 7 Mar 2018 19:52:00 +0400 Subject: [PATCH 17/44] Backporting MDEV-15497 Wrong empty value in a GEOMETRY column on LOAD DATA This is a part of "MDEV-18045 Backporting the MDEV-15497 changes to 10.2 branch" --- mysql-test/r/gis-loaddata.result | 25 +++++++++ mysql-test/r/loaddata.result | 34 ++++++++++++ mysql-test/std_data/loaddata/mdev-15497.txt | 1 + mysql-test/t/gis-loaddata.test | 21 ++++++++ mysql-test/t/loaddata.test | 24 +++++++++ sql/field.cc | 57 +++++++++++++++++++++ sql/field.h | 5 ++ sql/sql_load.cc | 40 +++------------ 8 files changed, 173 insertions(+), 34 deletions(-) create mode 100644 mysql-test/r/gis-loaddata.result create mode 100644 mysql-test/std_data/loaddata/mdev-15497.txt create mode 100644 mysql-test/t/gis-loaddata.test diff --git a/mysql-test/r/gis-loaddata.result b/mysql-test/r/gis-loaddata.result new file mode 100644 index 00000000000..1d25c0a496d --- /dev/null +++ b/mysql-test/r/gis-loaddata.result @@ -0,0 +1,25 @@ +# +# MDEV-15497 Wrong empty value in a GEOMETRY column on LOAD DATA +# +SET sql_mode=''; +CREATE TABLE t1 (id INT, a GEOMETRY NOT NULL); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +ERROR 22004: Column set to default value; NULL supplied to NOT NULL column 'a' at row 1 +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +ERROR 22004: Column set to default value; NULL supplied to NOT NULL column 'a' at row 1 +DROP TABLE t1; +CREATE TABLE t1 (id INT, a GEOMETRY); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +id a +1 NULL +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +id a +1 NULL +DROP TABLE t1; diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index eda4f12291a..ca17cfce76e 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -580,3 +580,37 @@ SELECT HEX(a) FROM t1; HEX(a) C3A4 DROP TABLE t1; +# +# MDEV-15497 Wrong empty value in a GEOMETRY column on LOAD DATA +# +SET sql_mode=''; +CREATE TABLE t1 (a CHAR(1), b CHAR(1) NOT NULL); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +a b +1 +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +a b +1 +DROP TABLE t1; +CREATE TABLE t1 (a CHAR(1), b CHAR(1)); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +a b +1 NULL +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +a b +1 +DROP TABLE t1; diff --git a/mysql-test/std_data/loaddata/mdev-15497.txt b/mysql-test/std_data/loaddata/mdev-15497.txt new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-15497.txt @@ -0,0 +1 @@ +1 diff --git a/mysql-test/t/gis-loaddata.test b/mysql-test/t/gis-loaddata.test new file mode 100644 index 00000000000..ad6a2c56b08 --- /dev/null +++ b/mysql-test/t/gis-loaddata.test @@ -0,0 +1,21 @@ +--echo # +--echo # MDEV-15497 Wrong empty value in a GEOMETRY column on LOAD DATA +--echo # + +SET sql_mode=''; + +CREATE TABLE t1 (id INT, a GEOMETRY NOT NULL); +--error ER_WARN_NULL_TO_NOTNULL +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +--error ER_WARN_NULL_TO_NOTNULL +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +DROP TABLE t1; + + +CREATE TABLE t1 (id INT, a GEOMETRY); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +SELECT * FROM t1; +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 4e355e5c3a7..722074b34d4 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -676,3 +676,27 @@ CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; SELECT HEX(a) FROM t1; DROP TABLE t1; + + +--echo # +--echo # MDEV-15497 Wrong empty value in a GEOMETRY column on LOAD DATA +--echo # + +SET sql_mode=''; + +CREATE TABLE t1 (a CHAR(1), b CHAR(1) NOT NULL); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +SELECT * FROM t1; +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +SELECT * FROM t1; +DROP TABLE t1; + + +CREATE TABLE t1 (a CHAR(1), b CHAR(1)); +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1; +SELECT * FROM t1; +TRUNCATE TABLE t1; +LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index cfd51da1487..18f99945645 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1361,6 +1361,26 @@ warn: } +/** + If a field does not have a corresponding data, it's behavior can vary: + - In case of the fixed file format + it's set to the default value for the data type, + such as 0 for numbers or '' for strings. + - In case of a non-fixed format + it's set to NULL for nullable fields, and + it's set to the default value for the data type for NOT NULL fields. + This seems to be by design. +*/ +bool Field::load_data_set_no_data(THD *thd, bool fixed_format) +{ + reset(); // Do not use the DEFAULT value + if (fixed_format) + set_notnull(); + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + bool Field::load_data_set_null(THD *thd) { reset(); @@ -1375,6 +1395,21 @@ bool Field::load_data_set_null(THD *thd) } +void Field::load_data_set_value(const char *pos, uint length, + CHARSET_INFO *cs) +{ + /* + Mark field as not null, we should do this for each row because of + restore_record... + */ + set_notnull(); + if (this == table->next_number_field) + table->auto_increment_field_not_null= true; + store(pos, length, cs); + set_has_explicit_value(); // Do not auto-update this field +} + + /** Numeric fields base class constructor. */ @@ -5363,6 +5398,22 @@ int Field_timestamp::set_time() } +bool Field_timestamp::load_data_set_no_data(THD *thd, bool fixed_format) +{ + if (!maybe_null()) + { + /* + Timestamp fields that are NOT NULL are autoupdated if there is no + corresponding value in the data file. + */ + set_time(); + set_has_explicit_value(); + return false; + } + return Field::load_data_set_no_data(thd, fixed_format); +} + + bool Field_timestamp::load_data_set_null(THD *thd) { if (!maybe_null()) @@ -8735,6 +8786,12 @@ bool Field_geom::can_optimize_range(const Item_bool_func *cond, } +bool Field_geom::load_data_set_no_data(THD *thd, bool fixed_format) +{ + return Field_geom::load_data_set_null(thd); +} + + bool Field_geom::load_data_set_null(THD *thd) { Field_blob::reset(); diff --git a/sql/field.h b/sql/field.h index 728b1f68880..c10262a68d0 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1145,6 +1145,9 @@ public: { return null_ptr != 0 || table->maybe_null; } // Set to NULL on LOAD DATA or LOAD XML virtual bool load_data_set_null(THD *thd); + // Reset when a LOAD DATA file ended unexpectedly + virtual bool load_data_set_no_data(THD *thd, bool fixed_format); + void load_data_set_value(const char *pos, uint length, CHARSET_INFO *cs); /* @return true if this field is NULL-able (even if temporarily) */ inline bool real_maybe_null(void) const { return null_ptr != 0; } @@ -2485,6 +2488,7 @@ public: return get_equal_const_item_datetime(thd, ctx, const_item); } bool load_data_set_null(THD *thd); + bool load_data_set_no_data(THD *thd, bool fixed_format); uint size_of() const { return sizeof(*this); } }; @@ -3506,6 +3510,7 @@ public: */ int reset(void) { return Field_blob::reset() || !maybe_null(); } bool load_data_set_null(THD *thd); + bool load_data_set_no_data(THD *thd, bool fixed_format); geometry_type get_geometry_type() { return geom_type; }; static geometry_type geometry_type_merge(geometry_type, geometry_type); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index da3aafa58e1..53b43ebb1f5 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -966,26 +966,15 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, { Field *field= sql_field->field; table->auto_increment_field_not_null= auto_increment_field_not_null; - /* - No fields specified in fields_vars list can be null in this format. - Mark field as not null, we should do this for each row because of - restore_record... - */ - field->set_notnull(); - if (pos == read_info.row_end) { + if (field->load_data_set_no_data(thd, true)) + DBUG_RETURN(1); thd->cuted_fields++; /* Not enough fields */ push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_TOO_FEW_RECORDS, ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), thd->get_stmt_da()->current_row_for_warning()); - /* - Timestamp fields that are NOT NULL are autoupdated if there is no - corresponding value in the data file. - */ - if (!field->maybe_null() && field->type() == FIELD_TYPE_TIMESTAMP) - field->set_time(); } else { @@ -995,13 +984,11 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, field->field_length) length=field->field_length; save_chr=pos[length]; pos[length]='\0'; // Safeguard aganst malloc - field->store((char*) pos,length,read_info.read_charset); + field->load_data_set_value((char*) pos,length,read_info.read_charset); pos[length]=save_chr; if ((pos+=length) > read_info.row_end) pos= read_info.row_end; /* Fills rest with space */ } - /* Do not auto-update this field. */ - field->set_has_explicit_value(); } if (pos != read_info.row_end) { @@ -1152,12 +1139,8 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, else { Field *field= real_item->field; - field->set_notnull(); read_info.row_end[0]=0; // Safe to change end marker - if (field == table->next_number_field) - table->auto_increment_field_not_null= TRUE; - field->store((char*) pos, length, read_info.read_charset); - field->set_has_explicit_value(); + field->load_data_set_value((char*) pos, length, read_info.read_charset); } } @@ -1192,15 +1175,8 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, else { Field *field= real_item->field; - if (field->reset()) - { - my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0),field->field_name, - thd->get_stmt_da()->current_row_for_warning()); + if (field->load_data_set_no_data(thd, false)) DBUG_RETURN(1); - } - if (!field->maybe_null() && field->type() == FIELD_TYPE_TIMESTAMP) - field->set_time(); - field->set_has_explicit_value(); /* TODO: We probably should not throw warning for each field. But how about intention to always have the same number @@ -1353,11 +1329,7 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, { Field *field= ((Item_field *)item)->field; - field->set_notnull(); - if (field == table->next_number_field) - table->auto_increment_field_not_null= TRUE; - field->store((char *) tag->value.ptr(), tag->value.length(), cs); - field->set_has_explicit_value(); + field->load_data_set_value(tag->value.ptr(), tag->value.length(), cs); } } From 80c3fd184d4eeb66cd520079c3d23595e52cfdc0 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 20 Mar 2018 13:02:44 +0400 Subject: [PATCH 18/44] Backporting MDEV-15597 Add class Load_data_outvar and avoid using Item::STRING_ITEM for Item_user_var_as_out_param detection This is a part of "MDEV-18045 Backporting the MDEV-15497 changes to 10.2 branch" --- mysql-test/r/loaddata.result | 21 ++ mysql-test/std_data/loaddata/nl.txt | 1 + mysql-test/t/loaddata.test | 16 ++ sql/field.cc | 18 ++ sql/item.cc | 25 +++ sql/item.h | 45 +++- sql/item_func.cc | 4 +- sql/item_func.h | 33 ++- sql/sql_load.cc | 323 +++++++++------------------- sql/structs.h | 36 ++++ 10 files changed, 294 insertions(+), 228 deletions(-) create mode 100644 mysql-test/std_data/loaddata/nl.txt diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index ca17cfce76e..4c5cee0aa25 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -614,3 +614,24 @@ SELECT * FROM t1; a b 1 DROP TABLE t1; +# +# MDEV-15597 Add class Load_data_outvar and avoid using Item::STRING_ITEM for Item_user_var_as_out_param detection +# +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (id integer not null auto_increment primary key); +LOAD DATA INFILE '../../std_data/loaddata/nl.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +id +0 +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (id integer not null auto_increment primary key); +LOAD DATA INFILE '../../std_data/loaddata/nl.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +Warnings: +Warning 1261 Row 1 doesn't contain data for all columns +SELECT * FROM t1; +id +1 +DROP TABLE t1; diff --git a/mysql-test/std_data/loaddata/nl.txt b/mysql-test/std_data/loaddata/nl.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/mysql-test/std_data/loaddata/nl.txt @@ -0,0 +1 @@ + diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 722074b34d4..a283e64d473 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -700,3 +700,19 @@ TRUNCATE TABLE t1; LOAD DATA INFILE '../../std_data/loaddata/mdev-15497.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; SELECT * FROM t1; DROP TABLE t1; + +--echo # +--echo # MDEV-15597 Add class Load_data_outvar and avoid using Item::STRING_ITEM for Item_user_var_as_out_param detection +--echo # + +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (id integer not null auto_increment primary key); +LOAD DATA INFILE '../../std_data/loaddata/nl.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +SELECT * FROM t1; +DROP TABLE t1; + +SET sql_mode=''; +CREATE TABLE t1 (id integer not null auto_increment primary key); +LOAD DATA INFILE '../../std_data/loaddata/nl.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index 18f99945645..080cf34c76d 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1375,7 +1375,25 @@ bool Field::load_data_set_no_data(THD *thd, bool fixed_format) { reset(); // Do not use the DEFAULT value if (fixed_format) + { set_notnull(); + /* + We're loading a fixed format file, e.g.: + LOAD DATA INFILE 't1.txt' INTO TABLE t1 FIELDS TERMINATED BY ''; + Suppose the file ended unexpectedly and no data was provided for an + auto-increment column in the current row. + Historically, if sql_mode=NO_AUTO_VALUE_ON_ZERO, then the column value + is set to 0 in such case (the next auto_increment value is not used). + This behaviour was introduced by the fix for "bug#12053" in mysql-4.1. + Note, loading a delimited file works differently: + "no data" is not converted to 0 on NO_AUTO_VALUE_ON_ZERO: + it's considered as equal to setting the column to NULL, + which is then replaced to the next auto_increment value. + This difference seems to be intentional. + */ + if (this == table->next_number_field) + table->auto_increment_field_not_null= true; + } set_has_explicit_value(); // Do not auto-update this field return false; } diff --git a/sql/item.cc b/sql/item.cc index b4958c68be6..46420fd2c78 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2645,6 +2645,31 @@ void Item_field::reset_field(Field *f) } +void Item_field::load_data_print_for_log_event(THD *thd, String *to) const +{ + append_identifier(thd, to, name, (uint) strlen(name)); +} + + +bool Item_field::load_data_set_no_data(THD *thd, const Load_data_param *param) +{ + if (field->load_data_set_no_data(thd, param->is_fixed_length())) + return true; + /* + TODO: We probably should not throw warning for each field. + But how about intention to always have the same number + of warnings in THD::cuted_fields (and get rid of cuted_fields + in the end ?) + */ + thd->cuted_fields++; + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_WARN_TOO_FEW_RECORDS, + ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), + thd->get_stmt_da()->current_row_for_warning()); + return false; +} + + bool Item_field::enumerate_field_refs_processor(void *arg) { Field_enumerator *fe= (Field_enumerator*)arg; diff --git a/sql/item.h b/sql/item.h index b941c57896c..ac79e3437fb 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1866,6 +1866,20 @@ public: { return 0; } + + virtual Load_data_outvar *get_load_data_outvar() + { + return 0; + } + Load_data_outvar *get_load_data_outvar_or_error() + { + Load_data_outvar *dst= get_load_data_outvar(); + if (dst) + return dst; + my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), name); + return NULL; + } + /** Test whether an expression is expensive to compute. Used during optimization to avoid computing expensive expressions during this @@ -2538,7 +2552,8 @@ public: }; -class Item_field :public Item_ident +class Item_field :public Item_ident, + public Load_data_outvar { protected: void set_field(Field *field); @@ -2585,6 +2600,30 @@ public: bool val_bool_result(); bool is_null_result(); bool send(Protocol *protocol, String *str_arg); + Load_data_outvar *get_load_data_outvar() + { + return this; + } + bool load_data_set_null(THD *thd, const Load_data_param *param) + { + return field->load_data_set_null(thd); + } + bool load_data_set_value(THD *thd, const char *pos, uint length, + const Load_data_param *param) + { + field->load_data_set_value(pos, length, param->charset()); + return false; + } + bool load_data_set_no_data(THD *thd, const Load_data_param *param); + void load_data_print_for_log_event(THD *thd, String *to) const; + bool load_data_add_outvar(THD *thd, Load_data_param *param) const + { + return param->add_outvar_field(thd, field); + } + uint load_data_fixed_length() const + { + return field->field_length; + } void reset_field(Field *f); bool fix_fields(THD *, Item **); void fix_after_pullout(st_select_lex *new_parent, Item **ref, bool merge); @@ -4379,6 +4418,10 @@ public: void cleanup(); Item_field *field_for_view_update() { return (*ref)->field_for_view_update(); } + Load_data_outvar *get_load_data_outvar() + { + return (*ref)->get_load_data_outvar(); + } virtual Ref_Type ref_type() { return REF; } // Row emulation: forwarding of ROW-related calls to ref diff --git a/sql/item_func.cc b/sql/item_func.cc index f755deea23b..28933aba7af 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -5751,7 +5751,9 @@ my_decimal* Item_user_var_as_out_param::val_decimal(my_decimal *decimal_buffer) } -void Item_user_var_as_out_param::print_for_load(THD *thd, String *str) +void Item_user_var_as_out_param::load_data_print_for_log_event(THD *thd, + String *str) + const { str->append('@'); append_identifier(thd, str, name.str, name.length); diff --git a/sql/item_func.h b/sql/item_func.h index fea8c28f74f..7565a8d5a08 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -2038,13 +2038,43 @@ public: in List and desire to place this code somewhere near other functions working with user variables. */ -class Item_user_var_as_out_param :public Item +class Item_user_var_as_out_param :public Item, + public Load_data_outvar { LEX_STRING name; user_var_entry *entry; public: Item_user_var_as_out_param(THD *thd, LEX_STRING a): Item(thd), name(a) { set_name(thd, a.str, 0, system_charset_info); } + Load_data_outvar *get_load_data_outvar() + { + return this; + } + bool load_data_set_null(THD *thd, const Load_data_param *param) + { + set_null_value(param->charset()); + return false; + } + bool load_data_set_no_data(THD *thd, const Load_data_param *param) + { + set_null_value(param->charset()); + return false; + } + bool load_data_set_value(THD *thd, const char *pos, uint length, + const Load_data_param *param) + { + set_value(pos, length, param->charset()); + return false; + } + void load_data_print_for_log_event(THD *thd, String *to) const; + bool load_data_add_outvar(THD *thd, Load_data_param *param) const + { + return param->add_outvar_user_var(thd); + } + uint load_data_fixed_length() const + { + return 0; + } /* We should return something different from FIELD_ITEM here */ enum Type type() const { return STRING_ITEM;} double val_real(); @@ -2053,7 +2083,6 @@ public: my_decimal *val_decimal(my_decimal *decimal_buffer); /* fix_fields() binds variable name with its entry structure */ bool fix_fields(THD *thd, Item **ref); - void print_for_load(THD *thd, String *str); void set_null_value(CHARSET_INFO* cs); void set_value(const char *str, uint length, CHARSET_INFO* cs); enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 53b43ebb1f5..37ac168de43 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -134,11 +134,10 @@ static bool wsrep_load_data_split(THD *thd, const TABLE *table, #define WSREP_LOAD_DATA_SPLIT(thd,table,info) /* empty */ #endif /* WITH_WSREP */ -class READ_INFO { +class READ_INFO: public Load_data_param +{ File file; String data; /* Read buffer */ - uint fixed_length; /* Length of the fixed length record */ - uint max_length; /* Max length of row */ Term_string m_field_term; /* FIELDS TERMINATED BY 'string' */ Term_string m_line_term; /* LINES TERMINATED BY 'string' */ Term_string m_line_start; /* LINES STARTING BY 'string' */ @@ -191,7 +190,7 @@ class READ_INFO { bool read_mbtail(String *str) { int chlen; - if ((chlen= my_charlen(read_charset, str->end() - 1, str->end())) == 1) + if ((chlen= my_charlen(charset(), str->end() - 1, str->end())) == 1) return false; // Single byte character found for (uint32 length0= str->length() - 1 ; MY_CS_IS_TOOSMALL(chlen); ) { @@ -202,7 +201,7 @@ class READ_INFO { return true; // EOF } str->append(chr); - chlen= my_charlen(read_charset, str->ptr() + length0, str->end()); + chlen= my_charlen(charset(), str->ptr() + length0, str->end()); if (chlen == MY_CS_ILSEQ) { /** @@ -224,10 +223,9 @@ public: bool error,line_cuted,found_null,enclosed; uchar *row_start, /* Found row starts here */ *row_end; /* Found row ends here */ - CHARSET_INFO *read_charset; LOAD_FILE_IO_CACHE cache; - READ_INFO(THD *thd, File file, uint tot_length, CHARSET_INFO *cs, + READ_INFO(THD *thd, File file, const Load_data_param ¶m, String &field_term,String &line_start,String &line_term, String &enclosed,int escape,bool get_it_from_net, bool is_fifo); ~READ_INFO(); @@ -282,6 +280,31 @@ static bool write_execute_load_query_log_event(THD *, sql_exchange*, const char*, const char*, bool, enum enum_duplicates, bool, bool, int); #endif /* EMBEDDED_LIBRARY */ + +bool Load_data_param::add_outvar_field(THD *thd, const Field *field) +{ + if (field->flags & BLOB_FLAG) + { + m_use_blobs= true; + m_fixed_length+= 256; // Will be extended if needed + } + else + m_fixed_length+= field->field_length; + return false; +} + + +bool Load_data_param::add_outvar_user_var(THD *thd) +{ + if (m_is_fixed_length) + { + my_error(ER_LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR, MYF(0)); + return true; + } + return false; +} + + /* Execute LOAD DATA query @@ -313,8 +336,6 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, File file; TABLE *table= NULL; int error= 0; - String *field_term=ex->field_term,*escaped=ex->escaped; - String *enclosed=ex->enclosed; bool is_fifo=0; #ifndef EMBEDDED_LIBRARY killed_state killed_status; @@ -343,7 +364,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, read_file_from_client = 0; //server is always in the same process #endif - if (escaped->length() > 1 || enclosed->length() > 1) + if (ex->escaped->length() > 1 || ex->enclosed->length() > 1) { my_message(ER_WRONG_FIELD_TERMINATORS, ER_THD(thd, ER_WRONG_FIELD_TERMINATORS), @@ -352,8 +373,8 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, } /* Report problems with non-ascii separators */ - if (!escaped->is_ascii() || !enclosed->is_ascii() || - !field_term->is_ascii() || + if (!ex->escaped->is_ascii() || !ex->enclosed->is_ascii() || + !ex->field_term->is_ascii() || !ex->line_term->is_ascii() || !ex->line_start->is_ascii()) { push_warning(thd, Sql_condition::WARN_LEVEL_WARN, @@ -450,39 +471,21 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, table->prepare_triggers_for_insert_stmt_or_event(); table->mark_columns_needed_for_insert(); - uint tot_length=0; - bool use_blobs= 0, use_vars= 0; + Load_data_param param(ex->cs ? ex->cs : thd->variables.collation_database, + !ex->field_term->length() && !ex->enclosed->length()); List_iterator_fast it(fields_vars); Item *item; while ((item= it++)) { - Item *real_item= item->real_item(); - - if (real_item->type() == Item::FIELD_ITEM) - { - Field *field= ((Item_field*)real_item)->field; - if (field->flags & BLOB_FLAG) - { - use_blobs= 1; - tot_length+= 256; // Will be extended if needed - } - else - tot_length+= field->field_length; - } - else if (item->type() == Item::STRING_ITEM) - use_vars= 1; + const Load_data_outvar *var= item->get_load_data_outvar_or_error(); + if (!var || var->load_data_add_outvar(thd, ¶m)) + DBUG_RETURN(true); } - if (use_blobs && !ex->line_term->length() && !field_term->length()) + if (param.use_blobs() && !ex->line_term->length() && !ex->field_term->length()) { my_message(ER_BLOBS_AND_NO_TERMINATED, - ER_THD(thd, ER_BLOBS_AND_NO_TERMINATED), - MYF(0)); - DBUG_RETURN(TRUE); - } - if (use_vars && !field_term->length() && !enclosed->length()) - { - my_error(ER_LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR, MYF(0)); + ER_THD(thd, ER_BLOBS_AND_NO_TERMINATED), MYF(0)); DBUG_RETURN(TRUE); } @@ -572,13 +575,13 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, bzero((char*) &info,sizeof(info)); info.ignore= ignore; info.handle_duplicates=handle_duplicates; - info.escape_char= (escaped->length() && (ex->escaped_given() || + info.escape_char= (ex->escaped->length() && (ex->escaped_given() || !(thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES))) - ? (*escaped)[0] : INT_MAX; + ? (*ex->escaped)[0] : INT_MAX; - READ_INFO read_info(thd, file, tot_length, - ex->cs ? ex->cs : thd->variables.collation_database, - *field_term,*ex->line_start, *ex->line_term, *enclosed, + READ_INFO read_info(thd, file, param, + *ex->field_term, *ex->line_start, + *ex->line_term, *ex->enclosed, info.escape_char, read_file_from_client, is_fifo); if (read_info.error) { @@ -639,14 +642,14 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, error= read_xml_field(thd, info, table_list, fields_vars, set_fields, set_values, read_info, *(ex->line_term), skip_lines, ignore); - else if (!field_term->length() && !enclosed->length()) + else if (read_info.is_fixed_length()) error= read_fixed_length(thd, info, table_list, fields_vars, set_fields, set_values, read_info, skip_lines, ignore); else error= read_sep_field(thd, info, table_list, fields_vars, set_fields, set_values, read_info, - *enclosed, skip_lines, ignore); + *ex->enclosed, skip_lines, ignore); thd_proc_info(thd, "End bulk insert"); if (!error) @@ -850,14 +853,9 @@ static bool write_execute_load_query_log_event(THD *thd, sql_exchange* ex, { if (n++) query_str.append(", "); - if (item->real_type() == Item::FIELD_ITEM) - append_identifier(thd, &query_str, item->name, strlen(item->name)); - else - { - /* Actually Item_user_var_as_out_param despite claiming STRING_ITEM. */ - DBUG_ASSERT(item->type() == Item::STRING_ITEM); - ((Item_user_var_as_out_param *)item)->print_for_load(thd, &query_str); - } + const Load_data_outvar *var= item->get_load_data_outvar(); + DBUG_ASSERT(var); + var->load_data_print_for_log_event(thd, &query_str); } query_str.append(")"); } @@ -905,9 +903,9 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, ulong skip_lines, bool ignore_check_option_errors) { List_iterator_fast it(fields_vars); - Item_field *sql_field; + Item *item; TABLE *table= table_list->table; - bool err, progress_reports, auto_increment_field_not_null=false; + bool err, progress_reports; ulonglong counter, time_to_report_progress; DBUG_ENTER("read_fixed_length"); @@ -917,12 +915,6 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, if ((thd->progress.max_counter= read_info.file_length()) == ~(my_off_t) 0) progress_reports= 0; - while ((sql_field= (Item_field*) it++)) - { - if (sql_field->field == table->next_number_field) - auto_increment_field_not_null= true; - } - while (!read_info.read_fixed_length()) { if (thd->killed) @@ -958,36 +950,27 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, #endif restore_record(table, s->default_values); - /* - There is no variables in fields_vars list in this format so - this conversion is safe. - */ - while ((sql_field= (Item_field*) it++)) + + while ((item= it++)) { - Field *field= sql_field->field; - table->auto_increment_field_not_null= auto_increment_field_not_null; + Load_data_outvar *dst= item->get_load_data_outvar(); + DBUG_ASSERT(dst); if (pos == read_info.row_end) { - if (field->load_data_set_no_data(thd, true)) + if (dst->load_data_set_no_data(thd, &read_info)) DBUG_RETURN(1); - thd->cuted_fields++; /* Not enough fields */ - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WARN_TOO_FEW_RECORDS, - ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), - thd->get_stmt_da()->current_row_for_warning()); } else { - uint length; - uchar save_chr; - if ((length=(uint) (read_info.row_end-pos)) > - field->field_length) - length=field->field_length; - save_chr=pos[length]; pos[length]='\0'; // Safeguard aganst malloc - field->load_data_set_value((char*) pos,length,read_info.read_charset); - pos[length]=save_chr; - if ((pos+=length) > read_info.row_end) - pos= read_info.row_end; /* Fills rest with space */ + uint length, fixed_length= dst->load_data_fixed_length(); + uchar save_chr; + if ((length=(uint) (read_info.row_end - pos)) > fixed_length) + length= fixed_length; + save_chr= pos[length]; pos[length]= '\0'; // Safeguard aganst malloc + dst->load_data_set_value(thd, (const char *) pos, length, &read_info); + pos[length]= save_chr; + if ((pos+= length) > read_info.row_end) + pos= read_info.row_end; // Fills rest with space } } if (pos != read_info.row_end) @@ -1087,8 +1070,6 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, { uint length; uchar *pos; - Item_field *real_item; - if (read_info.read_field()) break; @@ -1099,48 +1080,22 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, pos=read_info.row_start; length=(uint) (read_info.row_end-pos); - real_item= item->field_for_view_update(); + Load_data_outvar *dst= item->get_load_data_outvar_or_error(); + DBUG_ASSERT(dst); if ((!read_info.enclosed && (enclosed_length && length == 4 && !memcmp(pos, STRING_WITH_LEN("NULL")))) || (length == 1 && read_info.found_null)) { - if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_null_value( - read_info.read_charset); - } - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + if (dst->load_data_set_null(thd, &read_info)) DBUG_RETURN(1); - } - else - { - DBUG_ASSERT(real_item->field->table == table); - if (real_item->field->load_data_set_null(thd)) - DBUG_RETURN(1); - } - - continue; - } - - if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_value((char*) pos, length, - read_info.read_charset); - } - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); - DBUG_RETURN(1); } else { - Field *field= real_item->field; - read_info.row_end[0]=0; // Safe to change end marker - field->load_data_set_value((char*) pos, length, read_info.read_charset); + read_info.row_end[0]= 0; // Safe to change end marker + if (dst->load_data_set_value(thd, (const char *) pos, length, &read_info)) + DBUG_RETURN(1); } } @@ -1161,34 +1116,10 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, break; for (; item ; item= it++) { - Item_field *real_item= item->field_for_view_update(); - if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_null_value( - read_info.read_charset); - } - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + Load_data_outvar *dst= item->get_load_data_outvar_or_error(); + DBUG_ASSERT(dst); + if (dst->load_data_set_no_data(thd, &read_info)) DBUG_RETURN(1); - } - else - { - Field *field= real_item->field; - if (field->load_data_set_no_data(thd, false)) - DBUG_RETURN(1); - /* - TODO: We probably should not throw warning for each field. - But how about intention to always have the same number - of warnings in THD::cuted_fields (and get rid of cuted_fields - in the end ?) - */ - thd->cuted_fields++; - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WARN_TOO_FEW_RECORDS, - ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), - thd->get_stmt_da()->current_row_for_warning()); - } } } @@ -1250,7 +1181,6 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, Item *item; TABLE *table= table_list->table; bool no_trans_update_stmt; - CHARSET_INFO *cs= read_info.read_charset; DBUG_ENTER("read_xml_field"); no_trans_update_stmt= !table->file->has_transactions(); @@ -1296,41 +1226,14 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, while(tag && strcmp(tag->field.c_ptr(), item->name) != 0) tag= xmlit++; - - Item_field *real_item= item->field_for_view_update(); - if (!tag) // found null - { - if (item->type() == Item::STRING_ITEM) - ((Item_user_var_as_out_param *) item)->set_null_value(cs); - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); - DBUG_RETURN(1); - } - else - { - DBUG_ASSERT(real_item->field->table == table); - if (real_item->field->load_data_set_null(thd)) - DBUG_RETURN(1); - } - continue; - } - if (item->type() == Item::STRING_ITEM) - ((Item_user_var_as_out_param *) item)->set_value( - (char *) tag->value.ptr(), - tag->value.length(), cs); - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + Load_data_outvar *dst= item->get_load_data_outvar_or_error(); + DBUG_ASSERT(dst); + if (!tag ? dst->load_data_set_null(thd, &read_info) : + dst->load_data_set_value(thd, tag->value.ptr(), + tag->value.length(), + &read_info)) DBUG_RETURN(1); - } - else - { - - Field *field= ((Item_field *)item)->field; - field->load_data_set_value(tag->value.ptr(), tag->value.length(), cs); - } } if (read_info.error) @@ -1341,39 +1244,8 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, skip_lines--; continue; } - - if (item) - { - /* Have not read any field, thus input file is simply ended */ - if (item == fields_vars.head()) - break; - - for ( ; item; item= it++) - { - Item_field *real_item= item->field_for_view_update(); - if (item->type() == Item::STRING_ITEM) - ((Item_user_var_as_out_param *)item)->set_null_value(cs); - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); - DBUG_RETURN(1); - } - else - { - /* - QQ: We probably should not throw warning for each field. - But how about intention to always have the same number - of warnings in THD::cuted_fields (and get rid of cuted_fields - in the end ?) - */ - thd->cuted_fields++; - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WARN_TOO_FEW_RECORDS, - ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), - thd->get_stmt_da()->current_row_for_warning()); - } - } - } + + DBUG_ASSERT(!item); if (thd->killed || fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values, @@ -1435,14 +1307,16 @@ READ_INFO::unescape(char chr) */ -READ_INFO::READ_INFO(THD *thd, File file_par, uint tot_length, CHARSET_INFO *cs, +READ_INFO::READ_INFO(THD *thd, File file_par, + const Load_data_param ¶m, String &field_term, String &line_start, String &line_term, String &enclosed_par, int escape, bool get_it_from_net, bool is_fifo) - :file(file_par), fixed_length(tot_length), + :Load_data_param(param), + file(file_par), m_field_term(field_term), m_line_term(line_term), m_line_start(line_start), escape_char(escape), found_end_of_line(false), eof(false), - error(false), line_cuted(false), found_null(false), read_charset(cs) + error(false), line_cuted(false), found_null(false) { data.set_thread_specific(); /* @@ -1459,12 +1333,13 @@ READ_INFO::READ_INFO(THD *thd, File file_par, uint tot_length, CHARSET_INFO *cs, enclosed_char= enclosed_par.length() ? (uchar) enclosed_par[0] : INT_MAX; /* Set of a stack for unget if long terminators */ - uint length= MY_MAX(cs->mbmaxlen, MY_MAX(m_field_term.length(), - m_line_term.length())) + 1; + uint length= MY_MAX(charset()->mbmaxlen, MY_MAX(m_field_term.length(), + m_line_term.length())) + 1; set_if_bigger(length,line_start.length()); stack= stack_pos= (int*) thd->alloc(sizeof(int) * length); - if (data.reserve(tot_length)) + DBUG_ASSERT(m_fixed_length < UINT_MAX32); + if (data.reserve((size_t) m_fixed_length)) error=1; /* purecov: inspected */ else { @@ -1606,7 +1481,7 @@ int READ_INFO::read_field() for (;;) { // Make sure we have enough space for the longest multi-byte character. - while (data.length() + read_charset->mbmaxlen <= data.alloced_length()) + while (data.length() + charset()->mbmaxlen <= data.alloced_length()) { chr = GET; if (chr == my_b_EOF) @@ -1692,7 +1567,7 @@ int READ_INFO::read_field() } } data.append(chr); - if (use_mb(read_charset) && read_mbtail(&data)) + if (use_mb(charset()) && read_mbtail(&data)) goto found_eof; } /* @@ -1738,7 +1613,7 @@ int READ_INFO::read_fixed_length() return 1; } - for (data.length(0); data.length() < fixed_length ; ) + for (data.length(0); data.length() < m_fixed_length ; ) { if ((chr=GET) == my_b_EOF) goto found_eof; @@ -1791,8 +1666,8 @@ int READ_INFO::next_line() if (getbyte(&buf[0])) return 1; // EOF - if (use_mb(read_charset) && - (chlen= my_charlen(read_charset, buf, buf + 1)) != 1) + if (use_mb(charset()) && + (chlen= my_charlen(charset(), buf, buf + 1)) != 1) { uint i; for (i= 1; MY_CS_IS_TOOSMALL(chlen); ) @@ -1801,7 +1676,7 @@ int READ_INFO::next_line() DBUG_ASSERT(chlen != 1); if (getbyte(&buf[i++])) return 1; // EOF - chlen= my_charlen(read_charset, buf, buf + i); + chlen= my_charlen(charset(), buf, buf + i); } /* @@ -1972,7 +1847,7 @@ int READ_INFO::read_value(int delim, String *val) else { val->append(chr); - if (use_mb(read_charset) && read_mbtail(val)) + if (use_mb(charset()) && read_mbtail(val)) return my_b_EOF; } } diff --git a/sql/structs.h b/sql/structs.h index 98eb0f2585d..378b8c387e0 100644 --- a/sql/structs.h +++ b/sql/structs.h @@ -629,4 +629,40 @@ public: }; +class Load_data_param +{ +protected: + CHARSET_INFO *m_charset; // Character set of the file + ulonglong m_fixed_length; // Sum of target field lengths for fixed format + bool m_is_fixed_length; + bool m_use_blobs; +public: + Load_data_param(CHARSET_INFO *cs, bool is_fixed_length): + m_charset(cs), + m_fixed_length(0), + m_is_fixed_length(is_fixed_length), + m_use_blobs(false) + { } + bool add_outvar_field(THD *thd, const Field *field); + bool add_outvar_user_var(THD *thd); + CHARSET_INFO *charset() const { return m_charset; } + bool is_fixed_length() const { return m_is_fixed_length; } + bool use_blobs() const { return m_use_blobs; } +}; + + +class Load_data_outvar +{ +public: + virtual ~Load_data_outvar() {} + virtual bool load_data_set_null(THD *thd, const Load_data_param *param)= 0; + virtual bool load_data_set_value(THD *thd, const char *pos, uint length, + const Load_data_param *param)= 0; + virtual bool load_data_set_no_data(THD *thd, const Load_data_param *param)= 0; + virtual void load_data_print_for_log_event(THD *thd, class String *to) const= 0; + virtual bool load_data_add_outvar(THD *thd, Load_data_param *param) const= 0; + virtual uint load_data_fixed_length() const= 0; +}; + + #endif /* STRUCTS_INCLUDED */ From 243f829c1c772c1b8e9e0717fbf5839e84244559 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 15 Feb 2019 10:58:59 +0100 Subject: [PATCH 19/44] MDEV-9519: Data corruption will happen on the Galera cluster size change If we have a 2+ node cluster which is replicating from an async master and the binlog_format is set to STATEMENT and multi-row inserts are executed on a table with an auto_increment column such that values are automatically generated by MySQL, then the server node generates wrong auto_increment values, which are different from what was generated on the async master. In the title of the MDEV-9519 it was proposed to ban start slave on a Galera if master binlog_format = statement and wsrep_auto_increment_control = 1, but the problem can be solved without such a restriction. The causes and fixes: 1. We need to improve processing of changing the auto-increment values after changing the cluster size. 2. If wsrep auto_increment_control switched on during operation of the node, then we should immediately update the auto_increment_increment and auto_increment_offset global variables, without waiting of the next invocation of the wsrep_view_handler_cb() callback. In the current version these variables retain its initial values if wsrep_auto_increment_control is switched on during operation of the node, which leads to inconsistent results on the different nodes in some scenarios. 3. If wsrep auto_increment_control switched off during operation of the node, then we must return the original values of the auto_increment_increment and auto_increment_offset global variables, as the user has set. To make this possible, we need to add a "shadow copies" of these variables (which stores the latest values set by the user). https://jira.mariadb.org/browse/MDEV-9519 --- include/mysql/service_wsrep.h | 3 + mysql-test/suite/galera/disabled.def | 1 - .../r/galera_binlog_stmt_autoinc.result | 28 +++--- .../r/partition_auto_increment_max.result | 7 ++ .../parts/t/partition_auto_increment_max.test | 12 +++ sql/field.h | 49 ++++++++++ sql/ha_partition.cc | 44 +++++---- sql/handler.cc | 50 +++++++--- sql/mysqld.cc | 14 +++ sql/sql_class.h | 10 ++ sql/sql_insert.cc | 2 - sql/sql_plugin_services.ic | 1 + sql/sys_vars.cc | 93 ++++++++++++++++++- sql/wsrep_dummy.cc | 8 ++ sql/wsrep_thd.cc | 21 +++++ storage/innobase/handler/ha_innodb.cc | 65 ++++++++++--- storage/xtradb/handler/ha_innodb.cc | 65 ++++++++++--- 17 files changed, 392 insertions(+), 81 deletions(-) create mode 100644 mysql-test/suite/parts/r/partition_auto_increment_max.result create mode 100644 mysql-test/suite/parts/t/partition_auto_increment_max.test diff --git a/include/mysql/service_wsrep.h b/include/mysql/service_wsrep.h index b51f154422f..267c8cb4e90 100644 --- a/include/mysql/service_wsrep.h +++ b/include/mysql/service_wsrep.h @@ -107,6 +107,7 @@ extern struct wsrep_service_st { bool (*wsrep_thd_ignore_table_func)(THD *thd); long long (*wsrep_thd_trx_seqno_func)(THD *thd); struct wsrep_ws_handle * (*wsrep_thd_ws_handle_func)(THD *thd); + void (*wsrep_thd_auto_increment_variables_func)(THD *thd, unsigned long long *offset, unsigned long long *increment); int (*wsrep_trx_is_aborting_func)(MYSQL_THD thd); int (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD); void (*wsrep_unlock_rollback_func)(); @@ -149,6 +150,7 @@ extern struct wsrep_service_st { #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T) #define wsrep_thd_trx_seqno(T) wsrep_service->wsrep_thd_trx_seqno_func(T) #define wsrep_thd_ws_handle(T) wsrep_service->wsrep_thd_ws_handle_func(T) +#define wsrep_thd_auto_increment_variables(T,O,I) wsrep_service->wsrep_thd_auto_increment_variables_func(T,O,I) #define wsrep_trx_is_aborting(T) wsrep_service->wsrep_trx_is_aborting_func(T) #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2) #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func() @@ -201,6 +203,7 @@ my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync); my_bool wsrep_thd_is_wsrep(MYSQL_THD thd); struct wsrep *get_wsrep(); struct wsrep_ws_handle *wsrep_thd_ws_handle(THD *thd); +void wsrep_thd_auto_increment_variables(THD *thd, unsigned long long *offset, unsigned long long *increment); void wsrep_aborting_thd_enqueue(THD *thd); void wsrep_lock_rollback(); void wsrep_post_commit(THD* thd, bool all); diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 1d8f8ad5949..d0bc20b0de7 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -25,7 +25,6 @@ galera.MW-329 : wsrep_local_replays not stable MW-416 : MDEV-13549 Galera test failures MW-388 : MDEV-13549 Galera test failures galera_sst_mysqldump_with_key : MDEV-16890 Galera test failure -galera_binlog_stmt_autoinc: MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc galera_gc_fc_limit : MDEV-17061 Test failure on galera.galera_gc_fc_limit galera_as_slave_replication_budle : MDEV-15785 Test case galera_as_slave_replication_bundle caused debug assertion galera_wan : MDEV-17259: Test failure on galera.galera_wan diff --git a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result index 8e8b79b168f..542bd156816 100644 --- a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result +++ b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result @@ -8,20 +8,20 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; @@ -40,20 +40,20 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text SET GLOBAL wsrep_auto_increment_control='ON'; SET SESSION binlog_format='ROW'; show variables like 'binlog_format'; @@ -67,7 +67,7 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; @@ -82,20 +82,20 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; @@ -114,20 +114,20 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text SET GLOBAL wsrep_auto_increment_control='ON'; show variables like 'binlog_format'; Variable_name Value @@ -140,7 +140,7 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; diff --git a/mysql-test/suite/parts/r/partition_auto_increment_max.result b/mysql-test/suite/parts/r/partition_auto_increment_max.result new file mode 100644 index 00000000000..65a3900e8e6 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_auto_increment_max.result @@ -0,0 +1,7 @@ +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); +UPDATE t1 SET pk = 2147483647; +ERROR 23000: Duplicate entry '2147483647' for key 'PRIMARY' +REPLACE INTO t1 VALUES (NULL); +ERROR 22003: Out of range value for column 'pk' at row 1 +DROP TABLE t1; diff --git a/mysql-test/suite/parts/t/partition_auto_increment_max.test b/mysql-test/suite/parts/t/partition_auto_increment_max.test new file mode 100644 index 00000000000..74e6139131d --- /dev/null +++ b/mysql-test/suite/parts/t/partition_auto_increment_max.test @@ -0,0 +1,12 @@ +--source include/have_partition.inc +--source include/have_log_bin.inc + +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); + +--error ER_DUP_ENTRY +UPDATE t1 SET pk = 2147483647; +--error HA_ERR_AUTOINC_ERANGE +REPLACE INTO t1 VALUES (NULL); + +DROP TABLE t1; diff --git a/sql/field.h b/sql/field.h index c21bba6936f..86853f7d9d9 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1329,6 +1329,17 @@ public: /* Hash value */ virtual void hash(ulong *nr, ulong *nr2); + /** + Get the upper limit of the MySQL integral and floating-point type. + + @return maximum allowed value for the field + */ + virtual ulonglong get_max_int_value() const + { + DBUG_ASSERT(false); + return 0ULL; + } + /** Checks whether a string field is part of write_set. @@ -1796,6 +1807,11 @@ public: *to= *from; return from + 1; } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFULL : 0x7FULL; + } }; @@ -1837,6 +1853,11 @@ public: virtual const uchar *unpack(uchar* to, const uchar *from, const uchar *from_end, uint param_data) { return unpack_int16(to, from, from_end); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFULL : 0x7FFFULL; + } }; class Field_medium :public Field_num { @@ -1870,6 +1891,11 @@ public: { return Field::pack(to, from, max_length); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFULL : 0x7FFFFFULL; + } }; @@ -1915,6 +1941,11 @@ public: { return unpack_int32(to, from, from_end); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFULL : 0x7FFFFFFFULL; + } }; @@ -1964,6 +1995,10 @@ public: { return unpack_int64(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFFFFFFFFFULL : 0x7FFFFFFFFFFFFFFFULL; + } }; @@ -1997,6 +2032,13 @@ public: uint32 pack_length() const { return sizeof(float); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^24 + */ + return 0x1000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; @@ -2039,6 +2081,13 @@ public: uint32 pack_length() const { return sizeof(double); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^53 + */ + return 0x20000000000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index d0ae8f47007..2167bea8d7c 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -8706,31 +8706,37 @@ void ha_partition::release_auto_increment() m_file[i]->ha_release_auto_increment(); } } - else if (next_insert_id) + else { - ulonglong next_auto_inc_val; lock_auto_increment(); - next_auto_inc_val= part_share->next_auto_inc_val; - /* - If the current auto_increment values is lower than the reserved - value, and the reserved value was reserved by this thread, - we can lower the reserved value. - */ - if (next_insert_id < next_auto_inc_val && - auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + if (next_insert_id) { - THD *thd= ha_thd(); + ulonglong next_auto_inc_val= part_share->next_auto_inc_val; /* - Check that we do not lower the value because of a failed insert - with SET INSERT_ID, i.e. forced/non generated values. + If the current auto_increment values is lower than the reserved + value, and the reserved value was reserved by this thread, + we can lower the reserved value. */ - if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) - part_share->next_auto_inc_val= next_insert_id; + if (next_insert_id < next_auto_inc_val && + auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + { + THD *thd= ha_thd(); + /* + Check that we do not lower the value because of a failed insert + with SET INSERT_ID, i.e. forced/non generated values. + */ + if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) + part_share->next_auto_inc_val= next_insert_id; + } + DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", + (ulong) part_share->next_auto_inc_val)); } - DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", - (ulong) part_share->next_auto_inc_val)); - - /* Unlock the multi row statement lock taken in get_auto_increment */ + /* + Unlock the multi-row statement lock taken in get_auto_increment. + These actions must be performed even if the next_insert_id field + contains zero, otherwise if the update_auto_increment fails then + an unnecessary lock will remain: + */ if (auto_increment_safe_stmt_log_lock) { auto_increment_safe_stmt_log_lock= FALSE; diff --git a/sql/handler.cc b/sql/handler.cc index 497409d0f53..ba3f7f3e16a 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2854,11 +2854,17 @@ compute_next_insert_id(ulonglong nr,struct system_variables *variables) nr= nr + 1; // optimization of the formula below else { - nr= (((nr+ variables->auto_increment_increment - - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); - nr= (nr* (ulonglong) variables->auto_increment_increment + - variables->auto_increment_offset); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr + variables->auto_increment_increment - + variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ + nr= nr * (ulonglong) variables->auto_increment_increment + + variables->auto_increment_offset; } if (unlikely(nr <= save_nr)) @@ -2912,8 +2918,14 @@ prev_insert_id(ulonglong nr, struct system_variables *variables) } if (variables->auto_increment_increment == 1) return nr; // optimization of the formula below - nr= (((nr - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr - variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ return (nr * (ulonglong) variables->auto_increment_increment + variables->auto_increment_offset); } @@ -3004,7 +3016,7 @@ int handler::update_auto_increment() bool append= FALSE; THD *thd= table->in_use; struct system_variables *variables= &thd->variables; - int result=0, tmp; + int tmp; enum enum_check_fields save_count_cuted_fields; DBUG_ENTER("handler::update_auto_increment"); @@ -3135,10 +3147,23 @@ int handler::update_auto_increment() if (unlikely(tmp)) // Out of range value in store { /* - It's better to return an error here than getting a confusing - 'duplicate key error' later. + First, test if the query was aborted due to strict mode constraints + or new field value greater than maximum integer value: */ - result= HA_ERR_AUTOINC_ERANGE; + if (thd->killed == KILL_BAD_DATA || + nr > table->next_number_field->get_max_int_value()) + DBUG_RETURN(HA_ERR_AUTOINC_ERANGE); + /* + Field refused this value (overflow) and truncated it, use the result + of the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) + nr= table->next_number_field->val_int(); } if (append) { @@ -3163,9 +3188,6 @@ int handler::update_auto_increment() */ insert_id_for_cur_row= nr; - if (result) // overflow - DBUG_RETURN(result); - /* Set next insert id to point to next auto-increment value to be able to handle multi-row statements. diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5452f7164e6..813f0988b74 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4563,6 +4563,20 @@ static int init_common_variables() return 1; } +#ifdef WITH_WSREP + /* + We need to initialize auxiliary variables, that will be + further keep the original values of auto-increment options + as they set by the user. These variables used to restore + user-defined values of the auto-increment options after + setting of the wsrep_auto_increment_control to 'OFF'. + */ + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; +#endif /* WITH_WSREP */ + return 0; } diff --git a/sql/sql_class.h b/sql/sql_class.h index fae4a683370..8dde68b7fba 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -550,6 +550,16 @@ typedef struct system_variables ha_rows max_join_size; ha_rows expensive_subquery_limit; ulong auto_increment_increment, auto_increment_offset; +#ifdef WITH_WSREP + /* + Stored values of the auto_increment_increment and auto_increment_offset + that are will be restored when wsrep_auto_increment_control will be set + to 'OFF', because the setting it to 'ON' leads to overwriting of the + original values (which are set by the user) by calculated ones (which + are based on the cluster size): + */ + ulong saved_auto_increment_increment, saved_auto_increment_offset; +#endif /* WITH_WSREP */ ulong lock_wait_timeout; ulong join_cache_level; ulong max_allowed_packet; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index de5c3b4c409..d695c586529 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1862,7 +1862,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) info->deleted++; else error= 0; - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Since we pretend that we have done insert we should call its after triggers. @@ -1903,7 +1902,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) if (table->file->insert_id_for_cur_row == 0) table->file->insert_id_for_cur_row= insert_id_for_cur_row; - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Restore column maps if they where replaced during an duplicate key problem. diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 95301a5fbe8..427d8937c57 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -177,6 +177,7 @@ static struct wsrep_service_st wsrep_handler = { wsrep_thd_ignore_table, wsrep_thd_trx_seqno, wsrep_thd_ws_handle, + wsrep_thd_auto_increment_variables, wsrep_trx_is_aborting, wsrep_trx_order_before, wsrep_unlock_rollback, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d1db14d7b35..223015e81c2 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -345,13 +345,56 @@ static Sys_var_long Sys_pfs_connect_attrs_size( #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_increment (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + else + thd->variables.saved_auto_increment_increment= + thd->variables.auto_increment_increment; + return false; +} + +#endif /* WITH_WSREP */ + static Sys_var_ulong Sys_auto_increment_increment( "auto_increment_increment", "Auto-increment columns are incremented by this", SESSION_VAR(auto_increment_increment), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_increment)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ + +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_offset (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; + else + thd->variables.saved_auto_increment_offset= + thd->variables.auto_increment_offset; + return false; +} + +#endif /* WITH_WSREP */ static Sys_var_ulong Sys_auto_increment_offset( "auto_increment_offset", @@ -360,7 +403,12 @@ static Sys_var_ulong Sys_auto_increment_offset( SESSION_VAR(auto_increment_offset), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_offset)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ static Sys_var_mybool Sys_automatic_sp_privileges( "automatic_sp_privileges", @@ -4851,11 +4899,54 @@ static Sys_var_ulong Sys_wsrep_retry_autocommit( SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1)); +static bool update_wsrep_auto_increment_control (sys_var *self, THD *thd, enum_var_type type) +{ + if (wsrep_auto_increment_control) + { + /* + The variables that control auto increment shall be calculated + automaticaly based on the size of the cluster. This usually done + within the wsrep_view_handler_cb callback. However, if the user + manually sets the value of wsrep_auto_increment_control to 'ON', + then we should to re-calculate these variables again (because + these values may be required before wsrep_view_handler_cb will + be re-invoked, which is rarely invoked if the cluster stays in + the stable state): + */ + global_system_variables.auto_increment_increment= + wsrep_cluster_size ? wsrep_cluster_size : 1; + global_system_variables.auto_increment_offset= + wsrep_local_index >= 0 ? wsrep_local_index + 1 : 1; + thd->variables.auto_increment_increment= + global_system_variables.auto_increment_increment; + thd->variables.auto_increment_offset= + global_system_variables.auto_increment_offset; + } + else + { + /* + We must restore the last values of the variables that + are explicitly specified by the user: + */ + global_system_variables.auto_increment_increment= + global_system_variables.saved_auto_increment_increment; + global_system_variables.auto_increment_offset= + global_system_variables.saved_auto_increment_offset; + thd->variables.auto_increment_increment= + thd->variables.saved_auto_increment_increment; + thd->variables.auto_increment_offset= + thd->variables.saved_auto_increment_offset; + } + return false; +} + static Sys_var_mybool Sys_wsrep_auto_increment_control( "wsrep_auto_increment_control", "To automatically control the " "assignment of autoincrement variables", GLOBAL_VAR(wsrep_auto_increment_control), - CMD_LINE(OPT_ARG), DEFAULT(TRUE)); + CMD_LINE(OPT_ARG), DEFAULT(TRUE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_wsrep_auto_increment_control)); static Sys_var_mybool Sys_wsrep_drupal_282555_workaround( "wsrep_drupal_282555_workaround", "Enable a workaround to handle the " diff --git a/sql/wsrep_dummy.cc b/sql/wsrep_dummy.cc index 5837ab4bed5..795e2d19252 100644 --- a/sql/wsrep_dummy.cc +++ b/sql/wsrep_dummy.cc @@ -125,6 +125,14 @@ longlong wsrep_thd_trx_seqno(THD *) struct wsrep_ws_handle* wsrep_thd_ws_handle(THD *) { return 0; } +void wsrep_thd_auto_increment_variables(THD *thd, + unsigned long long *offset, + unsigned long long *increment) +{ + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; +} + int wsrep_trx_is_aborting(THD *) { return 0; } diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index eb26da61282..551e710cfeb 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -676,3 +676,24 @@ bool wsrep_thd_has_explicit_locks(THD *thd) assert(thd); return thd->mdl_context.has_explicit_locks(); } + +/* + Get auto increment variables for THD. Use global settings for + applier threads. + */ +void wsrep_thd_auto_increment_variables(THD* thd, + unsigned long long* offset, + unsigned long long* increment) +{ + if (thd->wsrep_exec_mode == REPL_RECV && + thd->wsrep_conflict_state != REPLAYING) + { + *offset= global_system_variables.auto_increment_offset; + *increment= global_system_variables.auto_increment_increment; + } + else + { + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; + } +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 472fb86288f..80ff2e02e13 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8324,8 +8324,8 @@ no_commit: /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); /* Get the value that MySQL attempted to store in the table.*/ auto_inc = table->next_number_field->val_uint(); @@ -8402,15 +8402,33 @@ set_max_autoinc: /* This should filter out the negative values set explicitly by the user. */ if (auto_inc <= col_max_value) { - ut_a(prebuilt->autoinc_increment > 0); ulonglong offset; ulonglong increment; dberr_t err; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(user_thd) && + wsrep_thd_exec_mode(user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + ut_a(prebuilt->autoinc_increment > 0); + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, @@ -8925,17 +8943,35 @@ ha_innobase::update_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); if (auto_inc <= col_max_value && auto_inc != 0) { ulonglong offset; ulonglong increment; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(user_thd) && + wsrep_thd_exec_mode(user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, col_max_value); @@ -16040,12 +16076,11 @@ ha_innobase::get_auto_increment( current, autoinc); if (!wsrep_on(ha_thd())) { - current = autoinc - prebuilt->autoinc_increment; + current = autoinc - prebuilt->autoinc_increment; + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); - dict_table_autoinc_initialize(prebuilt->table, current); *first_value = current; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index c9db941b4bf..4ed9f644b45 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -8911,8 +8911,8 @@ no_commit: /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); /* Get the value that MySQL attempted to store in the table.*/ auto_inc = table->next_number_field->val_uint(); @@ -8989,15 +8989,33 @@ set_max_autoinc: /* This should filter out the negative values set explicitly by the user. */ if (auto_inc <= col_max_value) { - ut_a(prebuilt->autoinc_increment > 0); ulonglong offset; ulonglong increment; dberr_t err; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(user_thd) && + wsrep_thd_exec_mode(user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + ut_a(prebuilt->autoinc_increment > 0); + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, @@ -9509,17 +9527,35 @@ ha_innobase::update_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); if (auto_inc <= col_max_value && auto_inc != 0) { ulonglong offset; ulonglong increment; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(user_thd) && + wsrep_thd_exec_mode(user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, col_max_value); @@ -16744,12 +16780,11 @@ ha_innobase::get_auto_increment( current, autoinc); if (!wsrep_on(ha_thd())) { - current = autoinc - prebuilt->autoinc_increment; + current = autoinc - prebuilt->autoinc_increment; + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); - dict_table_autoinc_initialize(prebuilt->table, current); *first_value = current; From 2c734c980ef0292e00fc26492f17eb412907688a Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 25 Feb 2019 23:28:46 +0100 Subject: [PATCH 20/44] MDEV-9519: Data corruption will happen on the Galera cluster size change If we have a 2+ node cluster which is replicating from an async master and the binlog_format is set to STATEMENT and multi-row inserts are executed on a table with an auto_increment column such that values are automatically generated by MySQL, then the server node generates wrong auto_increment values, which are different from what was generated on the async master. In the title of the MDEV-9519 it was proposed to ban start slave on a Galera if master binlog_format = statement and wsrep_auto_increment_control = 1, but the problem can be solved without such a restriction. The causes and fixes: 1. We need to improve processing of changing the auto-increment values after changing the cluster size. 2. If wsrep auto_increment_control switched on during operation of the node, then we should immediately update the auto_increment_increment and auto_increment_offset global variables, without waiting of the next invocation of the wsrep_view_handler_cb() callback. In the current version these variables retain its initial values if wsrep_auto_increment_control is switched on during operation of the node, which leads to inconsistent results on the different nodes in some scenarios. 3. If wsrep auto_increment_control switched off during operation of the node, then we must return the original values of the auto_increment_increment and auto_increment_offset global variables, as the user has set. To make this possible, we need to add a "shadow copies" of these variables (which stores the latest values set by the user). https://jira.mariadb.org/browse/MDEV-9519 --- include/mysql/service_wsrep.h | 3 + mysql-test/suite/galera/disabled.def | 2 - .../r/galera_binlog_stmt_autoinc.result | 42 ++++++--- .../r/partition_auto_increment_max.result | 7 ++ .../parts/t/partition_auto_increment_max.test | 12 +++ sql/field.h | 49 ++++++++++ sql/ha_partition.cc | 44 +++++---- sql/handler.cc | 50 +++++++--- sql/mysqld.cc | 14 +++ sql/sql_class.h | 10 ++ sql/sql_insert.cc | 2 - sql/sql_plugin_services.ic | 1 + sql/sys_vars.cc | 93 ++++++++++++++++++- sql/wsrep_dummy.cc | 8 ++ sql/wsrep_thd.cc | 21 +++++ storage/innobase/handler/ha_innodb.cc | 70 ++++++++++---- 16 files changed, 360 insertions(+), 68 deletions(-) create mode 100644 mysql-test/suite/parts/r/partition_auto_increment_max.result create mode 100644 mysql-test/suite/parts/t/partition_auto_increment_max.test diff --git a/include/mysql/service_wsrep.h b/include/mysql/service_wsrep.h index b51f154422f..267c8cb4e90 100644 --- a/include/mysql/service_wsrep.h +++ b/include/mysql/service_wsrep.h @@ -107,6 +107,7 @@ extern struct wsrep_service_st { bool (*wsrep_thd_ignore_table_func)(THD *thd); long long (*wsrep_thd_trx_seqno_func)(THD *thd); struct wsrep_ws_handle * (*wsrep_thd_ws_handle_func)(THD *thd); + void (*wsrep_thd_auto_increment_variables_func)(THD *thd, unsigned long long *offset, unsigned long long *increment); int (*wsrep_trx_is_aborting_func)(MYSQL_THD thd); int (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD); void (*wsrep_unlock_rollback_func)(); @@ -149,6 +150,7 @@ extern struct wsrep_service_st { #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T) #define wsrep_thd_trx_seqno(T) wsrep_service->wsrep_thd_trx_seqno_func(T) #define wsrep_thd_ws_handle(T) wsrep_service->wsrep_thd_ws_handle_func(T) +#define wsrep_thd_auto_increment_variables(T,O,I) wsrep_service->wsrep_thd_auto_increment_variables_func(T,O,I) #define wsrep_trx_is_aborting(T) wsrep_service->wsrep_trx_is_aborting_func(T) #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2) #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func() @@ -201,6 +203,7 @@ my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync); my_bool wsrep_thd_is_wsrep(MYSQL_THD thd); struct wsrep *get_wsrep(); struct wsrep_ws_handle *wsrep_thd_ws_handle(THD *thd); +void wsrep_thd_auto_increment_variables(THD *thd, unsigned long long *offset, unsigned long long *increment); void wsrep_aborting_thd_enqueue(THD *thd); void wsrep_lock_rollback(); void wsrep_post_commit(THD* thd, bool all); diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4dcfdec84ce..bcef3c2508b 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -30,10 +30,8 @@ galera_kill_applier : race condition at the start of the test galera_ist_progress: MDEV-15236 galera_ist_progress fails when trying to read transfer status pxc-421: Lock timeout exceeded galera_sst_mysqldump_with_key : MDEV-16890 Galera test failure -galera.galera_binlog_stmt_autoinc : MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc galera.galera_kill_ddl : MDEV-17108 Test failure on galera.galera_kill_ddl galera.galera_var_node_address : MDEV-17151 Galera test failure on galera.galera_var_node_address -galera_binlog_stmt_autoinc: MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc galera_gc_fc_limit : MDEV-17061 Test failure on galera.galera_gc_fc_limit galera_as_slave_replication_budle : MDEV-15785 Test case galera_as_slave_replication_bundle caused debug assertion galera_wan : MDEV-17259: Test failure on galera.galera_wan diff --git a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result index 8e8b79b168f..78b40228eb0 100644 --- a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result +++ b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result @@ -1,4 +1,8 @@ +connection node_1; +connection node_2; +connection node_2; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; CREATE TABLE t1 ( i int(11) NOT NULL AUTO_INCREMENT, @@ -8,21 +12,23 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; SET SESSION binlog_format='STATEMENT'; @@ -40,20 +46,22 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_1; SET GLOBAL wsrep_auto_increment_control='ON'; SET SESSION binlog_format='ROW'; show variables like 'binlog_format'; @@ -67,12 +75,14 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; drop table t1; +connection node_2; SET GLOBAL wsrep_forced_binlog_format='ROW'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='ROW'; CREATE TABLE t1 ( i int(11) NOT NULL AUTO_INCREMENT, @@ -82,21 +92,23 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; SET SESSION binlog_format='ROW'; @@ -114,20 +126,22 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_1; SET GLOBAL wsrep_auto_increment_control='ON'; show variables like 'binlog_format'; Variable_name Value @@ -140,7 +154,7 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; diff --git a/mysql-test/suite/parts/r/partition_auto_increment_max.result b/mysql-test/suite/parts/r/partition_auto_increment_max.result new file mode 100644 index 00000000000..65a3900e8e6 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_auto_increment_max.result @@ -0,0 +1,7 @@ +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); +UPDATE t1 SET pk = 2147483647; +ERROR 23000: Duplicate entry '2147483647' for key 'PRIMARY' +REPLACE INTO t1 VALUES (NULL); +ERROR 22003: Out of range value for column 'pk' at row 1 +DROP TABLE t1; diff --git a/mysql-test/suite/parts/t/partition_auto_increment_max.test b/mysql-test/suite/parts/t/partition_auto_increment_max.test new file mode 100644 index 00000000000..74e6139131d --- /dev/null +++ b/mysql-test/suite/parts/t/partition_auto_increment_max.test @@ -0,0 +1,12 @@ +--source include/have_partition.inc +--source include/have_log_bin.inc + +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); + +--error ER_DUP_ENTRY +UPDATE t1 SET pk = 2147483647; +--error HA_ERR_AUTOINC_ERANGE +REPLACE INTO t1 VALUES (NULL); + +DROP TABLE t1; diff --git a/sql/field.h b/sql/field.h index c10262a68d0..5a1ec2df8d0 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1454,6 +1454,17 @@ public: /* Hash value */ virtual void hash(ulong *nr, ulong *nr2); + /** + Get the upper limit of the MySQL integral and floating-point type. + + @return maximum allowed value for the field + */ + virtual ulonglong get_max_int_value() const + { + DBUG_ASSERT(false); + return 0ULL; + } + /** Checks whether a string field is part of write_set. @@ -2012,6 +2023,11 @@ public: *to= *from; return from + 1; } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFULL : 0x7FULL; + } }; @@ -2053,6 +2069,11 @@ public: virtual const uchar *unpack(uchar* to, const uchar *from, const uchar *from_end, uint param_data) { return unpack_int16(to, from, from_end); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFULL : 0x7FFFULL; + } }; class Field_medium :public Field_integer { @@ -2086,6 +2107,11 @@ public: { return Field::pack(to, from, max_length); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFULL : 0x7FFFFFULL; + } }; @@ -2131,6 +2157,11 @@ public: { return unpack_int32(to, from, from_end); } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFULL : 0x7FFFFFFFULL; + } }; @@ -2180,6 +2211,10 @@ public: { return unpack_int64(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFFFFFFFFFULL : 0x7FFFFFFFFFFFFFFFULL; + } }; @@ -2219,6 +2254,13 @@ public: uint32 pack_length() const { return sizeof(float); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^24 + */ + return 0x1000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; @@ -2271,6 +2313,13 @@ public: uint32 pack_length() const { return sizeof(double); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^53 + */ + return 0x20000000000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 3eabc698fed..8700610415c 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -8739,31 +8739,37 @@ void ha_partition::release_auto_increment() m_file[i]->ha_release_auto_increment(); } } - else if (next_insert_id) + else { - ulonglong next_auto_inc_val; lock_auto_increment(); - next_auto_inc_val= part_share->next_auto_inc_val; - /* - If the current auto_increment values is lower than the reserved - value, and the reserved value was reserved by this thread, - we can lower the reserved value. - */ - if (next_insert_id < next_auto_inc_val && - auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + if (next_insert_id) { - THD *thd= ha_thd(); + ulonglong next_auto_inc_val= part_share->next_auto_inc_val; /* - Check that we do not lower the value because of a failed insert - with SET INSERT_ID, i.e. forced/non generated values. + If the current auto_increment values is lower than the reserved + value, and the reserved value was reserved by this thread, + we can lower the reserved value. */ - if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) - part_share->next_auto_inc_val= next_insert_id; + if (next_insert_id < next_auto_inc_val && + auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + { + THD *thd= ha_thd(); + /* + Check that we do not lower the value because of a failed insert + with SET INSERT_ID, i.e. forced/non generated values. + */ + if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) + part_share->next_auto_inc_val= next_insert_id; + } + DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", + (ulong) part_share->next_auto_inc_val)); } - DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", - (ulong) part_share->next_auto_inc_val)); - - /* Unlock the multi row statement lock taken in get_auto_increment */ + /* + Unlock the multi-row statement lock taken in get_auto_increment. + These actions must be performed even if the next_insert_id field + contains zero, otherwise if the update_auto_increment fails then + an unnecessary lock will remain: + */ if (auto_increment_safe_stmt_log_lock) { auto_increment_safe_stmt_log_lock= FALSE; diff --git a/sql/handler.cc b/sql/handler.cc index a9688fb96b4..9faa3b61a81 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2902,11 +2902,17 @@ compute_next_insert_id(ulonglong nr,struct system_variables *variables) nr= nr + 1; // optimization of the formula below else { - nr= (((nr+ variables->auto_increment_increment - - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); - nr= (nr* (ulonglong) variables->auto_increment_increment + - variables->auto_increment_offset); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr + variables->auto_increment_increment - + variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ + nr= nr * (ulonglong) variables->auto_increment_increment + + variables->auto_increment_offset; } if (unlikely(nr <= save_nr)) @@ -2960,8 +2966,14 @@ prev_insert_id(ulonglong nr, struct system_variables *variables) } if (variables->auto_increment_increment == 1) return nr; // optimization of the formula below - nr= (((nr - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr - variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ return (nr * (ulonglong) variables->auto_increment_increment + variables->auto_increment_offset); } @@ -3052,7 +3064,7 @@ int handler::update_auto_increment() bool append= FALSE; THD *thd= table->in_use; struct system_variables *variables= &thd->variables; - int result=0, tmp; + int tmp; enum enum_check_fields save_count_cuted_fields; DBUG_ENTER("handler::update_auto_increment"); @@ -3184,10 +3196,23 @@ int handler::update_auto_increment() if (unlikely(tmp)) // Out of range value in store { /* - It's better to return an error here than getting a confusing - 'duplicate key error' later. + First, test if the query was aborted due to strict mode constraints + or new field value greater than maximum integer value: */ - result= HA_ERR_AUTOINC_ERANGE; + if (thd->killed == KILL_BAD_DATA || + nr > table->next_number_field->get_max_int_value()) + DBUG_RETURN(HA_ERR_AUTOINC_ERANGE); + /* + Field refused this value (overflow) and truncated it, use the result + of the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) + nr= table->next_number_field->val_int(); } if (append) { @@ -3212,9 +3237,6 @@ int handler::update_auto_increment() */ insert_id_for_cur_row= nr; - if (result) // overflow - DBUG_RETURN(result); - /* Set next insert id to point to next auto-increment value to be able to handle multi-row statements. diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9240622cadf..f4fd6658171 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4690,6 +4690,20 @@ static int init_common_variables() return 1; } +#ifdef WITH_WSREP + /* + We need to initialize auxiliary variables, that will be + further keep the original values of auto-increment options + as they set by the user. These variables used to restore + user-defined values of the auto-increment options after + setting of the wsrep_auto_increment_control to 'OFF'. + */ + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; +#endif /* WITH_WSREP */ + return 0; } diff --git a/sql/sql_class.h b/sql/sql_class.h index d678a20078a..7e05274cf06 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -569,6 +569,16 @@ typedef struct system_variables ha_rows max_join_size; ha_rows expensive_subquery_limit; ulong auto_increment_increment, auto_increment_offset; +#ifdef WITH_WSREP + /* + Stored values of the auto_increment_increment and auto_increment_offset + that are will be restored when wsrep_auto_increment_control will be set + to 'OFF', because the setting it to 'ON' leads to overwriting of the + original values (which are set by the user) by calculated ones (which + are based on the cluster size): + */ + ulong saved_auto_increment_increment, saved_auto_increment_offset; +#endif /* WITH_WSREP */ uint eq_range_index_dive_limit; ulong lock_wait_timeout; ulong join_cache_level; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 9821848bb2f..b6d31c27feb 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1877,7 +1877,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) info->deleted++; else error= 0; - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Since we pretend that we have done insert we should call its after triggers. @@ -1918,7 +1917,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) if (table->file->insert_id_for_cur_row == 0) table->file->insert_id_for_cur_row= insert_id_for_cur_row; - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Restore column maps if they where replaced during an duplicate key problem. diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 801e7d5e41f..7fb5524016a 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -177,6 +177,7 @@ static struct wsrep_service_st wsrep_handler = { wsrep_thd_ignore_table, wsrep_thd_trx_seqno, wsrep_thd_ws_handle, + wsrep_thd_auto_increment_variables, wsrep_trx_is_aborting, wsrep_trx_order_before, wsrep_unlock_rollback, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 9bb7436b2da..2a793646c7f 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -347,13 +347,56 @@ static Sys_var_long Sys_pfs_connect_attrs_size( #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_increment (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + else + thd->variables.saved_auto_increment_increment= + thd->variables.auto_increment_increment; + return false; +} + +#endif /* WITH_WSREP */ + static Sys_var_ulong Sys_auto_increment_increment( "auto_increment_increment", "Auto-increment columns are incremented by this", SESSION_VAR(auto_increment_increment), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_increment)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ + +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_offset (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; + else + thd->variables.saved_auto_increment_offset= + thd->variables.auto_increment_offset; + return false; +} + +#endif /* WITH_WSREP */ static Sys_var_ulong Sys_auto_increment_offset( "auto_increment_offset", @@ -362,7 +405,12 @@ static Sys_var_ulong Sys_auto_increment_offset( SESSION_VAR(auto_increment_offset), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_offset)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ static Sys_var_mybool Sys_automatic_sp_privileges( "automatic_sp_privileges", @@ -4980,11 +5028,54 @@ static Sys_var_ulong Sys_wsrep_retry_autocommit( SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1)); +static bool update_wsrep_auto_increment_control (sys_var *self, THD *thd, enum_var_type type) +{ + if (wsrep_auto_increment_control) + { + /* + The variables that control auto increment shall be calculated + automaticaly based on the size of the cluster. This usually done + within the wsrep_view_handler_cb callback. However, if the user + manually sets the value of wsrep_auto_increment_control to 'ON', + then we should to re-calculate these variables again (because + these values may be required before wsrep_view_handler_cb will + be re-invoked, which is rarely invoked if the cluster stays in + the stable state): + */ + global_system_variables.auto_increment_increment= + wsrep_cluster_size ? wsrep_cluster_size : 1; + global_system_variables.auto_increment_offset= + wsrep_local_index >= 0 ? wsrep_local_index + 1 : 1; + thd->variables.auto_increment_increment= + global_system_variables.auto_increment_increment; + thd->variables.auto_increment_offset= + global_system_variables.auto_increment_offset; + } + else + { + /* + We must restore the last values of the variables that + are explicitly specified by the user: + */ + global_system_variables.auto_increment_increment= + global_system_variables.saved_auto_increment_increment; + global_system_variables.auto_increment_offset= + global_system_variables.saved_auto_increment_offset; + thd->variables.auto_increment_increment= + thd->variables.saved_auto_increment_increment; + thd->variables.auto_increment_offset= + thd->variables.saved_auto_increment_offset; + } + return false; +} + static Sys_var_mybool Sys_wsrep_auto_increment_control( "wsrep_auto_increment_control", "To automatically control the " "assignment of autoincrement variables", GLOBAL_VAR(wsrep_auto_increment_control), - CMD_LINE(OPT_ARG), DEFAULT(TRUE)); + CMD_LINE(OPT_ARG), DEFAULT(TRUE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_wsrep_auto_increment_control)); static Sys_var_mybool Sys_wsrep_drupal_282555_workaround( "wsrep_drupal_282555_workaround", "Enable a workaround to handle the " diff --git a/sql/wsrep_dummy.cc b/sql/wsrep_dummy.cc index 5837ab4bed5..795e2d19252 100644 --- a/sql/wsrep_dummy.cc +++ b/sql/wsrep_dummy.cc @@ -125,6 +125,14 @@ longlong wsrep_thd_trx_seqno(THD *) struct wsrep_ws_handle* wsrep_thd_ws_handle(THD *) { return 0; } +void wsrep_thd_auto_increment_variables(THD *thd, + unsigned long long *offset, + unsigned long long *increment) +{ + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; +} + int wsrep_trx_is_aborting(THD *) { return 0; } diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index 15eed2e10e6..a3d1961ade2 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -677,3 +677,24 @@ bool wsrep_thd_has_explicit_locks(THD *thd) assert(thd); return thd->mdl_context.has_explicit_locks(); } + +/* + Get auto increment variables for THD. Use global settings for + applier threads. + */ +void wsrep_thd_auto_increment_variables(THD* thd, + unsigned long long* offset, + unsigned long long* increment) +{ + if (thd->wsrep_exec_mode == REPL_RECV && + thd->wsrep_conflict_state != REPLAYING) + { + *offset= global_system_variables.auto_increment_offset; + *increment= global_system_variables.auto_increment_increment; + } + else + { + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; + } +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 8641bedecba..840538657fe 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8253,8 +8253,8 @@ ha_innobase::write_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); /* Get the value that MySQL attempted to store in the table.*/ auto_inc = table->next_number_field->val_uint(); @@ -8329,15 +8329,32 @@ set_max_autoinc: /* This should filter out the negative values set explicitly by the user. */ if (auto_inc <= col_max_value) { - ut_a(m_prebuilt->autoinc_increment > 0); ulonglong offset; ulonglong increment; dberr_t err; - - offset = m_prebuilt->autoinc_offset; - increment = m_prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(m_user_thd) && + wsrep_thd_exec_mode(m_user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + m_user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + ut_a(m_prebuilt->autoinc_increment > 0); + offset = m_prebuilt->autoinc_offset; + increment = m_prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, @@ -9007,12 +9024,33 @@ ha_innobase::update_row( /* A value for an AUTO_INCREMENT column was specified in the UPDATE statement. */ + ulonglong offset; + ulonglong increment; +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(m_user_thd) && + wsrep_thd_exec_mode(m_user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + m_user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + offset = m_prebuilt->autoinc_offset; + increment = m_prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ + autoinc = innobase_next_autoinc( - autoinc, 1, - m_prebuilt->autoinc_increment, - m_prebuilt->autoinc_offset, - innobase_get_int_col_max_value( - table->found_next_number_field)); + autoinc, 1, increment, offset, + table->found_next_number_field->get_max_int_value()); error = innobase_set_max_autoinc(autoinc); @@ -16762,7 +16800,8 @@ ha_innobase::get_auto_increment( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - ulonglong col_max_value = innobase_get_int_col_max_value(table->next_number_field); + ulonglong col_max_value = + table->next_number_field->get_max_int_value(); /** The following logic is needed to avoid duplicate key error for autoincrement column. @@ -16843,11 +16882,10 @@ ha_innobase::get_auto_increment( if (!wsrep_on(m_user_thd)) { current = autoinc - m_prebuilt->autoinc_increment; + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); - dict_table_autoinc_initialize( m_prebuilt->table, current); From 50b3632fa434e6bf836830365f664b67ff989279 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 25 Feb 2019 21:49:04 +0100 Subject: [PATCH 21/44] MDEV-9519: Data corruption will happen on the Galera cluster size change If we have a 2+ node cluster which is replicating from an async master and the binlog_format is set to STATEMENT and multi-row inserts are executed on a table with an auto_increment column such that values are automatically generated by MySQL, then the server node generates wrong auto_increment values, which are different from what was generated on the async master. In the title of the MDEV-9519 it was proposed to ban start slave on a Galera if master binlog_format = statement and wsrep_auto_increment_control = 1, but the problem can be solved without such a restriction. The causes and fixes: 1. We need to improve processing of changing the auto-increment values after changing the cluster size. 2. If wsrep auto_increment_control switched on during operation of the node, then we should immediately update the auto_increment_increment and auto_increment_offset global variables, without waiting of the next invocation of the wsrep_view_handler_cb() callback. In the current version these variables retain its initial values if wsrep_auto_increment_control is switched on during operation of the node, which leads to inconsistent results on the different nodes in some scenarios. 3. If wsrep auto_increment_control switched off during operation of the node, then we must return the original values of the auto_increment_increment and auto_increment_offset global variables, as the user has set. To make this possible, we need to add a "shadow copies" of these variables (which stores the latest values set by the user). https://jira.mariadb.org/browse/MDEV-9519 --- include/mysql/service_wsrep.h | 3 + mysql-test/suite/galera/disabled.def | 2 - .../r/galera_binlog_stmt_autoinc.result | 42 ++++++--- .../r/partition_auto_increment_max.result | 7 ++ .../parts/t/partition_auto_increment_max.test | 12 +++ sql/field.h | 45 +++++++++ sql/ha_partition.cc | 44 +++++---- sql/handler.cc | 50 +++++++--- sql/mysqld.cc | 14 +++ sql/sql_class.h | 10 ++ sql/sql_insert.cc | 2 - sql/sql_plugin_services.ic | 1 + sql/sys_vars.cc | 93 ++++++++++++++++++- sql/wsrep_dummy.cc | 8 ++ sql/wsrep_thd.cc | 21 +++++ storage/innobase/handler/ha_innodb.cc | 70 ++++++++++---- 16 files changed, 356 insertions(+), 68 deletions(-) create mode 100644 mysql-test/suite/parts/r/partition_auto_increment_max.result create mode 100644 mysql-test/suite/parts/t/partition_auto_increment_max.test diff --git a/include/mysql/service_wsrep.h b/include/mysql/service_wsrep.h index cefba6303fe..461b1064633 100644 --- a/include/mysql/service_wsrep.h +++ b/include/mysql/service_wsrep.h @@ -109,6 +109,7 @@ extern struct wsrep_service_st { bool (*wsrep_thd_ignore_table_func)(THD *thd); long long (*wsrep_thd_trx_seqno_func)(THD *thd); struct wsrep_ws_handle * (*wsrep_thd_ws_handle_func)(THD *thd); + void (*wsrep_thd_auto_increment_variables_func)(THD *thd, unsigned long long *offset, unsigned long long *increment); int (*wsrep_trx_is_aborting_func)(MYSQL_THD thd); int (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD); void (*wsrep_unlock_rollback_func)(); @@ -153,6 +154,7 @@ extern struct wsrep_service_st { #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T) #define wsrep_thd_trx_seqno(T) wsrep_service->wsrep_thd_trx_seqno_func(T) #define wsrep_thd_ws_handle(T) wsrep_service->wsrep_thd_ws_handle_func(T) +#define wsrep_thd_auto_increment_variables(T,O,I) wsrep_service->wsrep_thd_auto_increment_variables_func(T,O,I) #define wsrep_trx_is_aborting(T) wsrep_service->wsrep_trx_is_aborting_func(T) #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2) #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func() @@ -207,6 +209,7 @@ my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync); my_bool wsrep_thd_is_wsrep(MYSQL_THD thd); struct wsrep *get_wsrep(); struct wsrep_ws_handle *wsrep_thd_ws_handle(THD *thd); +void wsrep_thd_auto_increment_variables(THD *thd, unsigned long long *offset, unsigned long long *increment); void wsrep_aborting_thd_enqueue(THD *thd); void wsrep_lock_rollback(); void wsrep_post_commit(THD* thd, bool all); diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 5f5142671f0..af3f576807e 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -34,10 +34,8 @@ galera_gcs_fc_limit : Timeouts pool_of_threads: WSREP has not yet prepared node for application use galera_var_innodb_disallow_writes : Timeout MW-336 : nondeterministic wsrep_thread_count -galera_binlog_stmt_autoinc : MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc galera_kill_ddl : MDEV-17108 Test failure on galera.galera_kill_ddl galera_var_node_address : MDEV-17151 Galera test failure on galera.galera_var_node_address -galera_binlog_stmt_autoinc: MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc galera_gc_fc_limit : MDEV-17061 Test failure on galera.galera_gc_fc_limit galera_as_slave_replication_budle : MDEV-15785 Test case galera_as_slave_replication_bundle caused debug assertion galera_wan : MDEV-17259: Test failure on galera.galera_wan diff --git a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result index 8e8b79b168f..78b40228eb0 100644 --- a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result +++ b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result @@ -1,4 +1,8 @@ +connection node_1; +connection node_2; +connection node_2; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; CREATE TABLE t1 ( i int(11) NOT NULL AUTO_INCREMENT, @@ -8,21 +12,23 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; SET SESSION binlog_format='STATEMENT'; @@ -40,20 +46,22 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_1; SET GLOBAL wsrep_auto_increment_control='ON'; SET SESSION binlog_format='ROW'; show variables like 'binlog_format'; @@ -67,12 +75,14 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; drop table t1; +connection node_2; SET GLOBAL wsrep_forced_binlog_format='ROW'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='ROW'; CREATE TABLE t1 ( i int(11) NOT NULL AUTO_INCREMENT, @@ -82,21 +92,23 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -3 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text -9 dummy_text SET GLOBAL wsrep_forced_binlog_format='none'; +connection node_1; SET GLOBAL wsrep_forced_binlog_format='none'; drop table t1; SET SESSION binlog_format='ROW'; @@ -114,20 +126,22 @@ PRIMARY KEY (i) insert into t1(i) values(null); select * from t1; i c -4 dummy_text +1 dummy_text insert into t1(i) values(null), (null), (null); select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_2; select * from t1; i c +1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text -13 dummy_text +connection node_1; SET GLOBAL wsrep_auto_increment_control='ON'; show variables like 'binlog_format'; Variable_name Value @@ -140,7 +154,7 @@ wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 2 +auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; diff --git a/mysql-test/suite/parts/r/partition_auto_increment_max.result b/mysql-test/suite/parts/r/partition_auto_increment_max.result new file mode 100644 index 00000000000..65a3900e8e6 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_auto_increment_max.result @@ -0,0 +1,7 @@ +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); +UPDATE t1 SET pk = 2147483647; +ERROR 23000: Duplicate entry '2147483647' for key 'PRIMARY' +REPLACE INTO t1 VALUES (NULL); +ERROR 22003: Out of range value for column 'pk' at row 1 +DROP TABLE t1; diff --git a/mysql-test/suite/parts/t/partition_auto_increment_max.test b/mysql-test/suite/parts/t/partition_auto_increment_max.test new file mode 100644 index 00000000000..74e6139131d --- /dev/null +++ b/mysql-test/suite/parts/t/partition_auto_increment_max.test @@ -0,0 +1,12 @@ +--source include/have_partition.inc +--source include/have_log_bin.inc + +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2; +INSERT INTO t1 VALUES (NULL),(NULL); + +--error ER_DUP_ENTRY +UPDATE t1 SET pk = 2147483647; +--error HA_ERR_AUTOINC_ERANGE +REPLACE INTO t1 VALUES (NULL); + +DROP TABLE t1; diff --git a/sql/field.h b/sql/field.h index 1d01dc72290..422a223ed87 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1495,6 +1495,17 @@ public: /* Hash value */ virtual void hash(ulong *nr, ulong *nr2); + /** + Get the upper limit of the MySQL integral and floating-point type. + + @return maximum allowed value for the field + */ + virtual ulonglong get_max_int_value() const + { + DBUG_ASSERT(false); + return 0ULL; + } + /** Checks whether a string field is part of write_set. @@ -2116,6 +2127,10 @@ public: *to= *from; return from + 1; } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFULL : 0x7FULL; + } }; @@ -2160,6 +2175,10 @@ public: virtual const uchar *unpack(uchar* to, const uchar *from, const uchar *from_end, uint param_data) { return unpack_int16(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFULL : 0x7FFFULL; + } }; class Field_medium :public Field_int @@ -2195,6 +2214,10 @@ public: { return Field::pack(to, from, max_length); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFULL : 0x7FFFFFULL; + } }; @@ -2244,6 +2267,10 @@ public: { return unpack_int32(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFULL : 0x7FFFFFFFULL; + } }; @@ -2298,6 +2325,10 @@ public: } void set_max(); bool is_max(); + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFFFFFFFFFULL : 0x7FFFFFFFFFFFFFFFULL; + } }; @@ -2380,6 +2411,13 @@ public: uint32 pack_length() const { return sizeof(float); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^24 + */ + return 0x1000000ULL; + } private: int save_field_metadata(uchar *first_byte); }; @@ -2433,6 +2471,13 @@ public: uint32 pack_length() const { return sizeof(double); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^53 + */ + return 0x20000000000000ULL; + } private: int save_field_metadata(uchar *first_byte); }; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 1349571a3f8..051a0e15af9 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -10511,31 +10511,37 @@ void ha_partition::release_auto_increment() m_file[i]->ha_release_auto_increment(); } } - else if (next_insert_id) + else { - ulonglong next_auto_inc_val; lock_auto_increment(); - next_auto_inc_val= part_share->next_auto_inc_val; - /* - If the current auto_increment values is lower than the reserved - value, and the reserved value was reserved by this thread, - we can lower the reserved value. - */ - if (next_insert_id < next_auto_inc_val && - auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + if (next_insert_id) { - THD *thd= ha_thd(); + ulonglong next_auto_inc_val= part_share->next_auto_inc_val; /* - Check that we do not lower the value because of a failed insert - with SET INSERT_ID, i.e. forced/non generated values. + If the current auto_increment values is lower than the reserved + value, and the reserved value was reserved by this thread, + we can lower the reserved value. */ - if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) - part_share->next_auto_inc_val= next_insert_id; + if (next_insert_id < next_auto_inc_val && + auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val) + { + THD *thd= ha_thd(); + /* + Check that we do not lower the value because of a failed insert + with SET INSERT_ID, i.e. forced/non generated values. + */ + if (thd->auto_inc_intervals_forced.maximum() < next_insert_id) + part_share->next_auto_inc_val= next_insert_id; + } + DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", + (ulong) part_share->next_auto_inc_val)); } - DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu", - (ulong) part_share->next_auto_inc_val)); - - /* Unlock the multi row statement lock taken in get_auto_increment */ + /* + Unlock the multi-row statement lock taken in get_auto_increment. + These actions must be performed even if the next_insert_id field + contains zero, otherwise if the update_auto_increment fails then + an unnecessary lock will remain: + */ if (auto_increment_safe_stmt_log_lock) { auto_increment_safe_stmt_log_lock= FALSE; diff --git a/sql/handler.cc b/sql/handler.cc index e0dd1fa9970..680bb49d5da 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3096,11 +3096,17 @@ compute_next_insert_id(ulonglong nr,struct system_variables *variables) nr= nr + 1; // optimization of the formula below else { - nr= (((nr+ variables->auto_increment_increment - - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); - nr= (nr* (ulonglong) variables->auto_increment_increment + - variables->auto_increment_offset); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr + variables->auto_increment_increment - + variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ + nr= nr * (ulonglong) variables->auto_increment_increment + + variables->auto_increment_offset; } if (unlikely(nr <= save_nr)) @@ -3154,8 +3160,14 @@ prev_insert_id(ulonglong nr, struct system_variables *variables) } if (variables->auto_increment_increment == 1) return nr; // optimization of the formula below - nr= (((nr - variables->auto_increment_offset)) / - (ulonglong) variables->auto_increment_increment); + /* + Calculating the number of complete auto_increment_increment extents: + */ + nr= (nr - variables->auto_increment_offset) / + (ulonglong) variables->auto_increment_increment; + /* + Adding an offset to the auto_increment_increment extent boundary: + */ return (nr * (ulonglong) variables->auto_increment_increment + variables->auto_increment_offset); } @@ -3246,7 +3258,7 @@ int handler::update_auto_increment() bool append= FALSE; THD *thd= table->in_use; struct system_variables *variables= &thd->variables; - int result=0, tmp; + int tmp; enum enum_check_fields save_count_cuted_fields; DBUG_ENTER("handler::update_auto_increment"); @@ -3397,10 +3409,23 @@ int handler::update_auto_increment() if (unlikely(tmp)) // Out of range value in store { /* - It's better to return an error here than getting a confusing - 'duplicate key error' later. + First, test if the query was aborted due to strict mode constraints + or new field value greater than maximum integer value: */ - result= HA_ERR_AUTOINC_ERANGE; + if (thd->killed == KILL_BAD_DATA || + nr > table->next_number_field->get_max_int_value()) + DBUG_RETURN(HA_ERR_AUTOINC_ERANGE); + /* + Field refused this value (overflow) and truncated it, use the result + of the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) + nr= table->next_number_field->val_int(); } if (append) { @@ -3425,9 +3450,6 @@ int handler::update_auto_increment() */ insert_id_for_cur_row= nr; - if (result) // overflow - DBUG_RETURN(result); - /* Set next insert id to point to next auto-increment value to be able to handle multi-row statements. diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a864b2dc257..35cfd669d19 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4762,6 +4762,20 @@ static int init_common_variables() global_system_variables.in_subquery_conversion_threshold= IN_SUBQUERY_CONVERSION_THRESHOLD; +#ifdef WITH_WSREP + /* + We need to initialize auxiliary variables, that will be + further keep the original values of auto-increment options + as they set by the user. These variables used to restore + user-defined values of the auto-increment options after + setting of the wsrep_auto_increment_control to 'OFF'. + */ + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; +#endif /* WITH_WSREP */ + return 0; } diff --git a/sql/sql_class.h b/sql/sql_class.h index b1fc89db7f2..02dec7e9499 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -581,6 +581,16 @@ typedef struct system_variables ha_rows max_join_size; ha_rows expensive_subquery_limit; ulong auto_increment_increment, auto_increment_offset; +#ifdef WITH_WSREP + /* + Stored values of the auto_increment_increment and auto_increment_offset + that are will be restored when wsrep_auto_increment_control will be set + to 'OFF', because the setting it to 'ON' leads to overwriting of the + original values (which are set by the user) by calculated ones (which + are based on the cluster size): + */ + ulong saved_auto_increment_increment, saved_auto_increment_offset; +#endif /* WITH_WSREP */ uint eq_range_index_dive_limit; ulong column_compression_zlib_strategy; ulong lock_wait_timeout; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 6e2fa5767f5..aade7e0d876 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1952,7 +1952,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) } else error= 0; // error was HA_ERR_RECORD_IS_THE_SAME - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Since we pretend that we have done insert we should call its after triggers. @@ -2008,7 +2007,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) if (table->file->insert_id_for_cur_row == 0) table->file->insert_id_for_cur_row= insert_id_for_cur_row; - thd->record_first_successful_insert_id_in_cur_stmt(table->file->insert_id_for_cur_row); /* Restore column maps if they where replaced during an duplicate key problem. diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 8de53aa2161..f0ec917d411 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -179,6 +179,7 @@ static struct wsrep_service_st wsrep_handler = { wsrep_thd_ignore_table, wsrep_thd_trx_seqno, wsrep_thd_ws_handle, + wsrep_thd_auto_increment_variables, wsrep_trx_is_aborting, wsrep_trx_order_before, wsrep_unlock_rollback, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index dadb4cdd9bb..ac245d1ee54 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -349,13 +349,56 @@ static Sys_var_long Sys_pfs_connect_attrs_size( #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_increment (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + else + thd->variables.saved_auto_increment_increment= + thd->variables.auto_increment_increment; + return false; +} + +#endif /* WITH_WSREP */ + static Sys_var_ulong Sys_auto_increment_increment( "auto_increment_increment", "Auto-increment columns are incremented by this", SESSION_VAR(auto_increment_increment), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_increment)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ + +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_offset (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; + else + thd->variables.saved_auto_increment_offset= + thd->variables.auto_increment_offset; + return false; +} + +#endif /* WITH_WSREP */ static Sys_var_ulong Sys_auto_increment_offset( "auto_increment_offset", @@ -364,7 +407,12 @@ static Sys_var_ulong Sys_auto_increment_offset( SESSION_VAR(auto_increment_offset), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_offset)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ static Sys_var_mybool Sys_automatic_sp_privileges( "automatic_sp_privileges", @@ -5339,11 +5387,54 @@ static Sys_var_ulong Sys_wsrep_retry_autocommit( SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1)); +static bool update_wsrep_auto_increment_control (sys_var *self, THD *thd, enum_var_type type) +{ + if (wsrep_auto_increment_control) + { + /* + The variables that control auto increment shall be calculated + automaticaly based on the size of the cluster. This usually done + within the wsrep_view_handler_cb callback. However, if the user + manually sets the value of wsrep_auto_increment_control to 'ON', + then we should to re-calculate these variables again (because + these values may be required before wsrep_view_handler_cb will + be re-invoked, which is rarely invoked if the cluster stays in + the stable state): + */ + global_system_variables.auto_increment_increment= + wsrep_cluster_size ? wsrep_cluster_size : 1; + global_system_variables.auto_increment_offset= + wsrep_local_index >= 0 ? wsrep_local_index + 1 : 1; + thd->variables.auto_increment_increment= + global_system_variables.auto_increment_increment; + thd->variables.auto_increment_offset= + global_system_variables.auto_increment_offset; + } + else + { + /* + We must restore the last values of the variables that + are explicitly specified by the user: + */ + global_system_variables.auto_increment_increment= + global_system_variables.saved_auto_increment_increment; + global_system_variables.auto_increment_offset= + global_system_variables.saved_auto_increment_offset; + thd->variables.auto_increment_increment= + thd->variables.saved_auto_increment_increment; + thd->variables.auto_increment_offset= + thd->variables.saved_auto_increment_offset; + } + return false; +} + static Sys_var_mybool Sys_wsrep_auto_increment_control( "wsrep_auto_increment_control", "To automatically control the " "assignment of autoincrement variables", GLOBAL_VAR(wsrep_auto_increment_control), - CMD_LINE(OPT_ARG), DEFAULT(TRUE)); + CMD_LINE(OPT_ARG), DEFAULT(TRUE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_wsrep_auto_increment_control)); static Sys_var_mybool Sys_wsrep_drupal_282555_workaround( "wsrep_drupal_282555_workaround", "Enable a workaround to handle the " diff --git a/sql/wsrep_dummy.cc b/sql/wsrep_dummy.cc index 9a4bbd01bcd..4438ede9c1c 100644 --- a/sql/wsrep_dummy.cc +++ b/sql/wsrep_dummy.cc @@ -134,6 +134,14 @@ longlong wsrep_thd_trx_seqno(THD *) struct wsrep_ws_handle* wsrep_thd_ws_handle(THD *) { return 0; } +void wsrep_thd_auto_increment_variables(THD *thd, + unsigned long long *offset, + unsigned long long *increment) +{ + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; +} + int wsrep_trx_is_aborting(THD *) { return 0; } diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index ce6d9688cb3..6d4e4f6bc5f 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -676,3 +676,24 @@ bool wsrep_thd_has_explicit_locks(THD *thd) assert(thd); return thd->mdl_context.has_explicit_locks(); } + +/* + Get auto increment variables for THD. Use global settings for + applier threads. + */ +void wsrep_thd_auto_increment_variables(THD* thd, + unsigned long long* offset, + unsigned long long* increment) +{ + if (thd->wsrep_exec_mode == REPL_RECV && + thd->wsrep_conflict_state != REPLAYING) + { + *offset= global_system_variables.auto_increment_offset; + *increment= global_system_variables.auto_increment_increment; + } + else + { + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; + } +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d0398bfe0e1..04d1288ee3b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8077,8 +8077,8 @@ ha_innobase::write_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); /* Get the value that MySQL attempted to store in the table.*/ auto_inc = table->next_number_field->val_uint(); @@ -8153,15 +8153,32 @@ set_max_autoinc: /* This should filter out the negative values set explicitly by the user. */ if (auto_inc <= col_max_value) { - ut_a(m_prebuilt->autoinc_increment > 0); ulonglong offset; ulonglong increment; dberr_t err; - - offset = m_prebuilt->autoinc_offset; - increment = m_prebuilt->autoinc_increment; - +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(m_user_thd) && + wsrep_thd_exec_mode(m_user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + m_user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + ut_a(m_prebuilt->autoinc_increment > 0); + offset = m_prebuilt->autoinc_offset; + increment = m_prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, @@ -8849,12 +8866,33 @@ ha_innobase::update_row( /* A value for an AUTO_INCREMENT column was specified in the UPDATE statement. */ + ulonglong offset; + ulonglong increment; +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(m_user_thd) && + wsrep_thd_exec_mode(m_user_thd) == REPL_RECV) + { + wsrep_thd_auto_increment_variables( + m_user_thd, &offset, &increment); + } + else + { +#endif /* WITH_WSREP */ + offset = m_prebuilt->autoinc_offset; + increment = m_prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ + autoinc = innobase_next_autoinc( - autoinc, 1, - m_prebuilt->autoinc_increment, - m_prebuilt->autoinc_offset, - innobase_get_int_col_max_value( - table->found_next_number_field)); + autoinc, 1, increment, offset, + table->found_next_number_field->get_max_int_value()); error = innobase_set_max_autoinc(autoinc); @@ -16479,7 +16517,8 @@ ha_innobase::get_auto_increment( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - ulonglong col_max_value = innobase_get_int_col_max_value(table->next_number_field); + ulonglong col_max_value = + table->next_number_field->get_max_int_value(); /** The following logic is needed to avoid duplicate key error for autoincrement column. @@ -16560,11 +16599,10 @@ ha_innobase::get_auto_increment( if (!wsrep_on(m_user_thd)) { current = autoinc - m_prebuilt->autoinc_increment; + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); - dict_table_autoinc_initialize( m_prebuilt->table, current); From 82da98556cf58f0fbb43c82e9c6ae1a887b6cf3d Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 25 Feb 2019 15:57:08 +0100 Subject: [PATCH 22/44] MDEV-18605: Loss of column aliases by using view and group Preserv column name with copy fields even if it is function and Co. --- mysql-test/main/view.result | 22 ++++++++++++++++++++++ mysql-test/main/view.test | 23 +++++++++++++++++++++++ sql/sql_select.cc | 2 ++ 3 files changed, 47 insertions(+) diff --git a/mysql-test/main/view.result b/mysql-test/main/view.result index d97517d5ce7..2629eb46200 100644 --- a/mysql-test/main/view.result +++ b/mysql-test/main/view.result @@ -6705,5 +6705,27 @@ drop table t1; ALTER VIEW IF NOT EXISTS v1 AS SELECT 1; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'IF NOT EXISTS v1 AS SELECT 1' at line 1 # +# MDEV-18605: Loss of column aliases by using view and group +# +CREATE TABLE t1 (id int, foo int); +CREATE VIEW v1 AS SELECT id, IFNULL(foo,'') AS foo FROM t1; +INSERT INTO t1 (id, foo) VALUES (1,1),(2,2); +SELECT v.id, v.foo AS bar FROM v1 v +WHERE id = 2; +id bar +2 2 +SELECT v.id, v.foo AS bar FROM v1 v +GROUP BY v.id; +id bar +1 1 +2 2 +SELECT v.id, v.foo AS bar FROM v1 v +WHERE id = 2 +GROUP BY v.id; +id bar +2 2 +Drop View v1; +Drop table t1; +# # End of 10.3 tests # diff --git a/mysql-test/main/view.test b/mysql-test/main/view.test index 82ef38f0eb6..643f050cee5 100644 --- a/mysql-test/main/view.test +++ b/mysql-test/main/view.test @@ -6414,6 +6414,29 @@ drop table t1; --error ER_PARSE_ERROR ALTER VIEW IF NOT EXISTS v1 AS SELECT 1; +--echo # +--echo # MDEV-18605: Loss of column aliases by using view and group +--echo # + +CREATE TABLE t1 (id int, foo int); +CREATE VIEW v1 AS SELECT id, IFNULL(foo,'') AS foo FROM t1; + +INSERT INTO t1 (id, foo) VALUES (1,1),(2,2); + +SELECT v.id, v.foo AS bar FROM v1 v + WHERE id = 2; + +SELECT v.id, v.foo AS bar FROM v1 v + GROUP BY v.id; + +SELECT v.id, v.foo AS bar FROM v1 v + WHERE id = 2 + GROUP BY v.id; + +#Cleanup +Drop View v1; +Drop table t1; + --echo # --echo # End of 10.3 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 439853c2f66..0bc27f18d47 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -23914,7 +23914,9 @@ setup_copy_fields(THD *thd, TMP_TABLE_PARAM *param, real_pos->type() == Item::COND_ITEM) && !real_pos->with_sum_func) { // Save for send fields + LEX_CSTRING real_name= pos->name; pos= real_pos; + pos->name= real_name; /* TODO: In most cases this result will be sent to the user. This should be changed to use copy_int or copy_real depending From cac14b92252b3e7bcbebb8090cdeb2ac47143d1d Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 26 Feb 2019 15:41:27 +0400 Subject: [PATCH 23/44] MDEV-17725 Assertion `!is_set() || (m_status == DA_OK_BULK && is_bulk_op())' failed in Diagnostics_area::set_ok_status upon ALTER failing due to error from engine --- mysql-test/suite/innodb/r/alter_table.result | 16 ++++++++++++++++ mysql-test/suite/innodb/t/alter_table.test | 16 ++++++++++++++++ sql/sql_table.cc | 3 +++ 3 files changed, 35 insertions(+) diff --git a/mysql-test/suite/innodb/r/alter_table.result b/mysql-test/suite/innodb/r/alter_table.result index 8a0717aa677..2fb09e2fe5c 100644 --- a/mysql-test/suite/innodb/r/alter_table.result +++ b/mysql-test/suite/innodb/r/alter_table.result @@ -22,3 +22,19 @@ alter table t1 change column id2 id4 varchar(100) not null; select * from t1 where id4 like 'a'; id1 id4 id3 drop table t1; +# +# MDEV-17725 Assertion `!is_set() || (m_status == DA_OK_BULK && is_bulk_op())' failed in Diagnostics_area::set_ok_status upon ALTER failing due to error from engine +# +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB; +ALTER TABLE t1 ORDER BY a; +Warnings: +Warning 1105 ORDER BY ignored as there is a user-defined clustered index in the table 't1' +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB; +ALTER TABLE t1 ORDER BY a; +Warnings: +Warning 1105 ORDER BY ignored as there is a user-defined clustered index in the table 't1' +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/mysql-test/suite/innodb/t/alter_table.test b/mysql-test/suite/innodb/t/alter_table.test index 474035436e5..ece24eb45c7 100644 --- a/mysql-test/suite/innodb/t/alter_table.test +++ b/mysql-test/suite/innodb/t/alter_table.test @@ -22,3 +22,19 @@ create table t1 ( alter table t1 change column id2 id4 varchar(100) not null; select * from t1 where id4 like 'a'; drop table t1; + +--echo # +--echo # MDEV-17725 Assertion `!is_set() || (m_status == DA_OK_BULK && is_bulk_op())' failed in Diagnostics_area::set_ok_status upon ALTER failing due to error from engine +--echo # + +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB; +ALTER TABLE t1 ORDER BY a; +DROP TABLE t1; + +SET sql_mode=''; +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB; +ALTER TABLE t1 ORDER BY a; +DROP TABLE t1; + +SET sql_mode=DEFAULT; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 1227f144348..af40ae23a6c 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -10021,11 +10021,14 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, to->file->ha_table_flags() & HA_TABLE_SCAN_ON_INDEX) { char warn_buff[MYSQL_ERRMSG_SIZE]; + bool save_abort_on_warning= thd->abort_on_warning; + thd->abort_on_warning= false; my_snprintf(warn_buff, sizeof(warn_buff), "ORDER BY ignored as there is a user-defined clustered index" " in the table '%-.192s'", from->s->table_name.str); push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_UNKNOWN_ERROR, warn_buff); + thd->abort_on_warning= save_abort_on_warning; } else { From 5a87e3ee874d04325bc6f106c9862c96a6a35172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 28 Feb 2019 09:29:19 +0200 Subject: [PATCH 24/44] Revert offending part of MDEV-9519: Data corruption will happen on the Galera cluster size change This will allow test binlog.binlog_stm_binlog to pass more often. Note that this is not a real fix to that test failure. --- sql/handler.cc | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index ba3f7f3e16a..f307b3895bc 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3016,7 +3016,7 @@ int handler::update_auto_increment() bool append= FALSE; THD *thd= table->in_use; struct system_variables *variables= &thd->variables; - int tmp; + int result=0, tmp; enum enum_check_fields save_count_cuted_fields; DBUG_ENTER("handler::update_auto_increment"); @@ -3152,18 +3152,27 @@ int handler::update_auto_increment() */ if (thd->killed == KILL_BAD_DATA || nr > table->next_number_field->get_max_int_value()) - DBUG_RETURN(HA_ERR_AUTOINC_ERANGE); - /* - Field refused this value (overflow) and truncated it, use the result - of the truncation (which is going to be inserted); however we try to - decrease it to honour auto_increment_* variables. - That will shift the left bound of the reserved interval, we don't - bother shifting the right bound (anyway any other value from this - interval will cause a duplicate key). - */ - nr= prev_insert_id(table->next_number_field->val_int(), variables); - if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) - nr= table->next_number_field->val_int(); + { + /* + It's better to return an error here than getting a confusing + 'duplicate key error' later. + */ + result= HA_ERR_AUTOINC_ERANGE; + } + else + { + /* + Field refused this value (overflow) and truncated it, use the result + of the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) + nr= table->next_number_field->val_int(); + } } if (append) { @@ -3188,6 +3197,9 @@ int handler::update_auto_increment() */ insert_id_for_cur_row= nr; + if (result) // overflow + DBUG_RETURN(result); + /* Set next insert id to point to next auto-increment value to be able to handle multi-row statements. From 622e9e8a7a5e9babe1dd41e7499c20f396b6ebdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 27 Feb 2019 13:15:03 +0200 Subject: [PATCH 25/44] MDEV-18265: Replace deprecated variable debug to debug_dbug on Galera tests Replaced debug to debug_dbug on 10.1 on galera suite. Nothing to do in wsrep and galera_3nodes suites. --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/MW-388.result | 12 +++------- mysql-test/suite/galera/r/MW-86-wait1.result | 11 ++------- mysql-test/suite/galera/r/MW-86-wait8.result | 11 ++------- .../suite/galera/r/galera_mdl_race.result | 16 ++++--------- .../r/galera_query_cache_sync_wait.result | 24 +++++-------------- mysql-test/suite/galera/t/MW-388.test | 10 ++++---- mysql-test/suite/galera/t/MW-86-wait1.test | 5 ++-- mysql-test/suite/galera/t/MW-86-wait8.test | 5 ++-- .../suite/galera/t/galera_mdl_race.test | 17 ++++--------- .../t/galera_query_cache_sync_wait.test | 13 +++++----- 11 files changed, 37 insertions(+), 88 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index d0bc20b0de7..b81c9a737ec 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -23,7 +23,6 @@ galera_as_slave_replication_bundle : MDEV-13549 Galera test failures galera_ssl_upgrade : MDEV-13549 Galera test failures galera.MW-329 : wsrep_local_replays not stable MW-416 : MDEV-13549 Galera test failures -MW-388 : MDEV-13549 Galera test failures galera_sst_mysqldump_with_key : MDEV-16890 Galera test failure galera_gc_fc_limit : MDEV-17061 Test failure on galera.galera_gc_fc_limit galera_as_slave_replication_budle : MDEV-15785 Test case galera_as_slave_replication_bundle caused debug assertion diff --git a/mysql-test/suite/galera/r/MW-388.result b/mysql-test/suite/galera/r/MW-388.result index 59d4d4a2bf3..141189e45c6 100644 --- a/mysql-test/suite/galera/r/MW-388.result +++ b/mysql-test/suite/galera/r/MW-388.result @@ -9,18 +9,14 @@ INSERT INTO t1 VALUES (1, 'node 1'),(2, 'node 1'); INSERT INTO t1 VALUES (3, 'node 1'); END| SET GLOBAL wsrep_slave_threads = 2; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; INSERT INTO t1 VALUES (1, 'node 2');; SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; SET SESSION wsrep_sync_wait = 0; SET SESSION DEBUG_SYNC = 'wsrep_after_replication SIGNAL wsrep_after_replication_reached WAIT_FOR wsrep_after_replication_continue'; CALL insert_proc ();; SET SESSION DEBUG_SYNC = "now WAIT_FOR wsrep_after_replication_reached"; -SET GLOBAL DEBUG = ""; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL wsrep_after_replication_continue"; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; SELECT @errno = 1213; @@ -37,9 +33,7 @@ f1 f2 SET GLOBAL wsrep_slave_threads = DEFAULT; DROP TABLE t1; DROP PROCEDURE insert_proc; -SET GLOBAL debug = NULL; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET debug_sync='RESET'; SELECT @@debug_sync; @@debug_sync diff --git a/mysql-test/suite/galera/r/MW-86-wait1.result b/mysql-test/suite/galera/r/MW-86-wait1.result index a38255eff8f..88b6ca30884 100644 --- a/mysql-test/suite/galera/r/MW-86-wait1.result +++ b/mysql-test/suite/galera/r/MW-86-wait1.result @@ -2,9 +2,7 @@ SELECT @@debug_sync; @@debug_sync ON - current signal: '' SET SESSION wsrep_sync_wait = 1; -SET GLOBAL debug = "+d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; CREATE TABLE t_wait1 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t_wait1 VALUES (1); SET SESSION debug_sync = "now WAIT_FOR sync.wsrep_apply_cb_reached"; @@ -33,15 +31,10 @@ SHOW TABLES; SHOW TRIGGERS; SHOW GLOBAL VARIABLES LIKE 'foo_bar'; SHOW WARNINGS; -SET GLOBAL debug = "-d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET SESSION debug_sync = "now SIGNAL signal.wsrep_apply_cb"; SET SESSION wsrep_sync_wait = default; DROP TABLE t_wait1; -SET GLOBAL debug = NULL; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead SET debug_sync='RESET'; SELECT @@debug_sync; @@debug_sync diff --git a/mysql-test/suite/galera/r/MW-86-wait8.result b/mysql-test/suite/galera/r/MW-86-wait8.result index 04c93e9a9f2..d5aa5037676 100644 --- a/mysql-test/suite/galera/r/MW-86-wait8.result +++ b/mysql-test/suite/galera/r/MW-86-wait8.result @@ -2,9 +2,7 @@ SELECT @@debug_sync; @@debug_sync ON - current signal: '' SET SESSION wsrep_sync_wait = 8; -SET GLOBAL debug = "+d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; CREATE TABLE t_wait8 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t_wait8 VALUES (1); SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT0.1S"; @@ -35,15 +33,10 @@ SHOW TABLES; SHOW TRIGGERS; SHOW GLOBAL VARIABLES LIKE 'foo_bar'; SHOW WARNINGS; -SET GLOBAL debug = "-d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; SET SESSION wsrep_sync_wait = default; DROP TABLE t_wait8; -SET GLOBAL debug = NULL; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead SET debug_sync='RESET'; SELECT @@debug_sync; @@debug_sync diff --git a/mysql-test/suite/galera/r/galera_mdl_race.result b/mysql-test/suite/galera/r/galera_mdl_race.result index 535f20de7f1..b48900669cf 100644 --- a/mysql-test/suite/galera/r/galera_mdl_race.result +++ b/mysql-test/suite/galera/r/galera_mdl_race.result @@ -1,22 +1,16 @@ -CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); -CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) engine=innodb; +CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) engine=innodb; INSERT INTO t1 VALUES (1, 'a'); INSERT INTO t1 VALUES (2, 'a'); SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE t1 SET f2 = 'b' WHERE f1 = 1; LOCK TABLE t2 WRITE; -SET GLOBAL DEBUG = "d,sync.wsrep_before_mdl_wait"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET @@debug_dbug = "d,sync.wsrep_before_mdl_wait"; SELECT * FROM t2;; -SET GLOBAL DEBUG = "d,sync.wsrep_after_BF_victim_lock"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET @@debug_dbug = "d,sync.wsrep_after_BF_victim_lock"; UPDATE t1 SET f2 = 'c' WHERE f1 = 1; -SET GLOBAL DEBUG = ""; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET @@debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_before_mdl_wait"; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_after_BF_victim_lock"; UNLOCK TABLES; diff --git a/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result b/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result index 5fe382048fb..b1cceebb010 100644 --- a/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result +++ b/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result @@ -1,30 +1,22 @@ CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S"; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; SELECT MAX(id) FROM t1; MAX(id) 1 INSERT INTO t1 VALUES (2); SELECT MAX(id) FROM t1; ERROR HY000: Lock wait timeout exceeded; try restarting transaction -SET GLOBAL DEBUG = ""; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; FLUSH QUERY CACHE; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; SET DEBUG_SYNC = "RESET"; INSERT INTO t1 VALUES (3); SELECT MAX(id) FROM t1; ERROR HY000: Lock wait timeout exceeded; try restarting transaction -SET GLOBAL DEBUG = ""; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; INSERT INTO t1 VALUES (4); SELECT MAX(id) FROM t1; @@ -37,14 +29,10 @@ MAX(id) SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'Qcache_hits'; VARIABLE_VALUE = 1 1 -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; INSERT INTO t1 VALUES (5); SELECT MAX(id) FROM t1 ; -SET GLOBAL DEBUG = ""; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; MAX(id) 5 diff --git a/mysql-test/suite/galera/t/MW-388.test b/mysql-test/suite/galera/t/MW-388.test index 209695dca80..fafdde092bf 100644 --- a/mysql-test/suite/galera/t/MW-388.test +++ b/mysql-test/suite/galera/t/MW-388.test @@ -1,6 +1,4 @@ --source include/galera_cluster.inc ---source include/have_innodb.inc ---source include/have_debug.inc --source include/have_debug_sync.inc --connection node_1 @@ -29,8 +27,9 @@ DELIMITER ;| # that of the INSERT. Because there is only one slave thread, # commit cut is not processed and therefore does not advance # local monitor, and our INSERT remains stuck there. + SET GLOBAL wsrep_slave_threads = 2; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; --connection node_2 --send INSERT INTO t1 VALUES (1, 'node 2'); @@ -47,8 +46,7 @@ SET SESSION DEBUG_SYNC = 'wsrep_after_replication SIGNAL wsrep_after_replication --connection node_1a SET SESSION DEBUG_SYNC = "now WAIT_FOR wsrep_after_replication_reached"; - -SET GLOBAL DEBUG = ""; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL wsrep_after_replication_continue"; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; @@ -69,7 +67,7 @@ SET GLOBAL wsrep_slave_threads = DEFAULT; DROP TABLE t1; DROP PROCEDURE insert_proc; -SET GLOBAL debug = NULL; +SET GLOBAL debug_dbug = ""; SET debug_sync='RESET'; # Make sure no pending signals are leftover to surprise subsequent tests. diff --git a/mysql-test/suite/galera/t/MW-86-wait1.test b/mysql-test/suite/galera/t/MW-86-wait1.test index 6c0982ad8b3..40a7882829b 100644 --- a/mysql-test/suite/galera/t/MW-86-wait1.test +++ b/mysql-test/suite/galera/t/MW-86-wait1.test @@ -12,7 +12,7 @@ SELECT @@debug_sync; SET SESSION wsrep_sync_wait = 1; -SET GLOBAL debug = "+d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; --connection node_1 CREATE TABLE t_wait1 (f1 INTEGER) ENGINE=InnoDB; @@ -90,7 +90,7 @@ SHOW WARNINGS; --enable_result_log # Unblock the background INSERT and remove the sync point. -SET GLOBAL debug = "-d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = ""; SET SESSION debug_sync = "now SIGNAL signal.wsrep_apply_cb"; SET SESSION wsrep_sync_wait = default; @@ -99,7 +99,6 @@ SET SESSION wsrep_sync_wait = default; # from the test. DROP TABLE t_wait1; -SET GLOBAL debug = NULL; SET debug_sync='RESET'; # Make sure no pending signals are leftover to surprise subsequent tests. diff --git a/mysql-test/suite/galera/t/MW-86-wait8.test b/mysql-test/suite/galera/t/MW-86-wait8.test index 65e612c5c8e..551b0f67b7c 100644 --- a/mysql-test/suite/galera/t/MW-86-wait8.test +++ b/mysql-test/suite/galera/t/MW-86-wait8.test @@ -10,7 +10,7 @@ SELECT @@debug_sync; SET SESSION wsrep_sync_wait = 8; -SET GLOBAL debug = "+d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; --connection node_1 CREATE TABLE t_wait8 (f1 INTEGER) ENGINE=InnoDB; @@ -112,7 +112,7 @@ SHOW WARNINGS; --enable_query_log # Unblock the background INSERT and remove the sync point. -SET GLOBAL debug = "-d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; SET SESSION wsrep_sync_wait = default; @@ -121,7 +121,6 @@ SET SESSION wsrep_sync_wait = default; # from the test. DROP TABLE t_wait8; -SET GLOBAL debug = NULL; SET debug_sync='RESET'; # Make sure no pending signals are leftover to surprise subsequent tests. diff --git a/mysql-test/suite/galera/t/galera_mdl_race.test b/mysql-test/suite/galera/t/galera_mdl_race.test index 508b85add4b..ad6770f9991 100644 --- a/mysql-test/suite/galera/t/galera_mdl_race.test +++ b/mysql-test/suite/galera/t/galera_mdl_race.test @@ -3,11 +3,10 @@ # --source include/galera_cluster.inc ---source include/have_innodb.inc --source include/have_debug_sync.inc -CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); -CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) engine=innodb; +CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) engine=innodb; INSERT INTO t1 VALUES (1, 'a'); INSERT INTO t1 VALUES (2, 'a'); @@ -24,7 +23,7 @@ LOCK TABLE t2 WRITE; # Block before MLD lock wait --connection node_1 - SET GLOBAL DEBUG = "d,sync.wsrep_before_mdl_wait"; + SET @@debug_dbug = "d,sync.wsrep_before_mdl_wait"; --send SELECT * FROM t2; # Wait for SELECT to be blocked @@ -35,27 +34,21 @@ LOCK TABLE t2 WRITE; #--source include/wait_condition.inc # block applier to wait after BF victim is locked -SET GLOBAL DEBUG = "d,sync.wsrep_after_BF_victim_lock"; +SET @@debug_dbug = "d,sync.wsrep_after_BF_victim_lock"; # Issue a conflicting update on node #2 --connection node_2 UPDATE t1 SET f2 = 'c' WHERE f1 = 1; ---sleep 3 - # Unblock the SELECT, to enter wsrep_thd_is_BF --connection node_1a -SET GLOBAL DEBUG = ""; +SET @@debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_before_mdl_wait"; ---sleep 3 - # unblock applier to try to BF the SELECT SET DEBUG_SYNC = "now SIGNAL signal.wsrep_after_BF_victim_lock"; - # table lock is not needed anymore ---sleep 3 UNLOCK TABLES; # SELECT succeeds diff --git a/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test b/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test index 87afc2d7bb2..e13e7f1f748 100644 --- a/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test +++ b/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test @@ -1,5 +1,4 @@ --source include/galera_cluster.inc ---source include/have_innodb.inc --source include/have_debug_sync.inc --source include/have_query_cache.inc @@ -9,7 +8,7 @@ INSERT INTO t1 VALUES (1); --connection node_2 --let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options` SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S"; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; SELECT MAX(id) FROM t1; # first lookup miss # @@ -22,11 +21,11 @@ INSERT INTO t1 VALUES (2); --connection node_2 --error ER_LOCK_WAIT_TIMEOUT SELECT MAX(id) FROM t1; -SET GLOBAL DEBUG = ""; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; FLUSH QUERY CACHE; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; SET DEBUG_SYNC = "RESET"; # @@ -39,7 +38,7 @@ INSERT INTO t1 VALUES (3); --connection node_2 --error ER_LOCK_WAIT_TIMEOUT SELECT MAX(id) FROM t1; -SET GLOBAL DEBUG = ""; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; # @@ -59,7 +58,7 @@ SELECT MAX(id) FROM t1; FLUSH STATUS; SELECT MAX(id) FROM t1; SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'Qcache_hits'; -SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb"; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; # # Query cache invalidated @@ -73,7 +72,7 @@ INSERT INTO t1 VALUES (5); --connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 --connection node_2a -SET GLOBAL DEBUG = ""; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; --connection node_2 From 0ad598a00b17008b0c0702db40948b14d7eee0d5 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 28 Feb 2019 18:13:28 +0400 Subject: [PATCH 26/44] A cleanup in derived table handling: removing duplicate code from st_select_lex::handle_derived() st_select_lex::handle_derived() and mysql_handle_list_of_derived() had exactly the same implementations. - Adding a new method LEX::handle_list_of_derived() instead - Removing public function mysql_handle_list_of_derived() - Reusing LEX::handle_list_of_derived() in st_select_lex::handle_derived() --- sql/sql_delete.cc | 6 +++--- sql/sql_derived.cc | 30 ------------------------------ sql/sql_derived.h | 1 - sql/sql_insert.cc | 2 +- sql/sql_lex.cc | 9 +-------- sql/sql_lex.h | 25 +++++++++++++++++++++++++ sql/sql_load.cc | 2 +- 7 files changed, 31 insertions(+), 44 deletions(-) diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index cdd7350cb0c..ab573df7992 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -37,7 +37,7 @@ #include "sql_trigger.h" #include "transaction.h" #include "records.h" // init_read_record, -#include "sql_derived.h" // mysql_handle_list_of_derived +#include "sql_derived.h" // mysql_handle_derived // end_read_record /** Implement DELETE SQL word. @@ -71,9 +71,9 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, if (open_and_lock_tables(thd, table_list, TRUE, 0)) DBUG_RETURN(TRUE); - if (mysql_handle_list_of_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT)) + if (thd->lex->handle_list_of_derived(table_list, DT_MERGE_FOR_INSERT)) DBUG_RETURN(TRUE); - if (mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE)) + if (thd->lex->handle_list_of_derived(table_list, DT_PREPARE)) DBUG_RETURN(TRUE); if (!table_list->single_table_updatable()) diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index 2e947ecba16..e990eba8e54 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -199,36 +199,6 @@ mysql_handle_single_derived(LEX *lex, TABLE_LIST *derived, uint phases) } -/** - Run specified phases for derived tables/views in the given list - - @param lex LEX for this thread - @param table_list list of derived tables/view to handle - @param phase_map phases to process tables/views through - - @details - This function runs phases specified by the 'phases_map' on derived - tables/views found in the 'dt_list' with help of the - TABLE_LIST::handle_derived function. - 'lex' is passed as an argument to the TABLE_LIST::handle_derived. - - @return FALSE ok - @return TRUE error -*/ - -bool -mysql_handle_list_of_derived(LEX *lex, TABLE_LIST *table_list, uint phases) -{ - for (TABLE_LIST *tl= table_list; tl; tl= tl->next_local) - { - if (tl->is_view_or_derived() && - tl->handle_derived(lex, phases)) - return TRUE; - } - return FALSE; -} - - /** Merge a derived table/view into the embedding select diff --git a/sql/sql_derived.h b/sql/sql_derived.h index f232445879e..29b898fc65a 100644 --- a/sql/sql_derived.h +++ b/sql/sql_derived.h @@ -22,7 +22,6 @@ struct LEX; bool mysql_handle_derived(LEX *lex, uint phases); bool mysql_handle_single_derived(LEX *lex, TABLE_LIST *derived, uint phases); -bool mysql_handle_list_of_derived(LEX *lex, TABLE_LIST *dt_list, uint phases); /** Cleans up the SELECT_LEX_UNIT for the derived table (if any). diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 809059093c2..ba8134bfaf1 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1404,7 +1404,7 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, DBUG_RETURN(TRUE); if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT)) DBUG_RETURN(TRUE); - if (mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE)) + if (thd->lex->handle_list_of_derived(table_list, DT_PREPARE)) DBUG_RETURN(TRUE); /* For subqueries in VALUES() we should not see the table in which we are diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index cfbde25314b..3c0d3049ae4 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -3593,14 +3593,7 @@ bool st_select_lex::optimize_unflattened_subqueries(bool const_only) bool st_select_lex::handle_derived(LEX *lex, uint phases) { - for (TABLE_LIST *cursor= (TABLE_LIST*) table_list.first; - cursor; - cursor= cursor->next_local) - { - if (cursor->is_view_or_derived() && cursor->handle_derived(lex, phases)) - return TRUE; - } - return FALSE; + return lex->handle_list_of_derived(table_list.first, phases); } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 57129cfedc7..8a31560d2bf 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -2785,6 +2785,31 @@ struct LEX: public Query_tables_list } bool save_prep_leaf_tables(); + + /* + Run specified phases for derived tables/views in the given list + + @param table_list - list of derived tables/view to handle + @param phase - phases to process tables/views through + + @details + This method runs phases specified by the 'phases' on derived + tables/views found in the 'table_list' with help of the + TABLE_LIST::handle_derived function. + 'this' is passed as an argument to the TABLE_LIST::handle_derived. + + @return false - ok + @return true - error + */ + bool handle_list_of_derived(TABLE_LIST *table_list, uint phases) + { + for (TABLE_LIST *tl= table_list; tl; tl= tl->next_local) + { + if (tl->is_view_or_derived() && tl->handle_derived(this, phases)) + return true; + } + return false; + } }; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 113cbe1dac0..850360c1dba 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -235,7 +235,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, DBUG_RETURN(TRUE); if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT)) DBUG_RETURN(TRUE); - if (mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE)) + if (thd->lex->handle_list_of_derived(table_list, DT_PREPARE)) DBUG_RETURN(TRUE); if (setup_tables_and_check_access(thd, &thd->lex->select_lex.context, &thd->lex->select_lex.top_join_list, From cb11b3fbe9d4dde776cb8f2c0d6f83a569655efc Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 27 Feb 2019 15:53:25 +0100 Subject: [PATCH 27/44] MDEV-17055: Server crashes in find_order_in_list upon 2nd (3rd) execution of SP with UPDATE 1. Always drop merged_for_insert flag on cleanup (there could be errors which prevent TABLE to be assigned) 2. Make more precise cleanup of select parts which was touched --- mysql-test/r/sp.result | 33 +++++++++++++++++++++++++++++++++ mysql-test/t/sp.test | 41 +++++++++++++++++++++++++++++++++++++++++ sql/sql_derived.cc | 4 ++-- sql/sql_insert.cc | 1 - sql/sql_lex.cc | 7 ++++--- sql/sql_lex.h | 7 ++++++- sql/sql_prepare.cc | 23 ++++++++++++++++------- sql/sql_union.cc | 5 +++-- 8 files changed, 105 insertions(+), 16 deletions(-) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 4535056242a..52b52fbbd24 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8072,4 +8072,37 @@ CALL sp; c a b a b DROP PROCEDURE sp; DROP TABLE t1; +# +# MDEV-17055: Server crashes in find_order_in_list upon +# 2nd (3rd) execution of SP with UPDATE +# +CREATE TABLE t1 (a INT); +CREATE VIEW v1 AS SELECT * FROM t1; +CREATE TABLE t2 (c INT); +CREATE PROCEDURE sp() UPDATE v1 SET a = 1 ORDER BY a, b LIMIT 1; +LOCK TABLE t2 READ; +CALL sp; +ERROR HY000: Table 'v1' was not locked with LOCK TABLES +UNLOCK TABLES; +CALL sp; +ERROR 42S22: Unknown column 'b' in 'order clause' +CALL sp; +ERROR 42S22: Unknown column 'b' in 'order clause' +CALL sp; +ERROR 42S22: Unknown column 'b' in 'order clause' +DROP PROCEDURE sp; +CREATE PROCEDURE sp() UPDATE v1 SET a = 1 WHERE a=1 and b=2; +LOCK TABLE t2 READ; +CALL sp; +ERROR HY000: Table 'v1' was not locked with LOCK TABLES +UNLOCK TABLES; +CALL sp; +ERROR 42S22: Unknown column 'b' in 'where clause' +CALL sp; +ERROR 42S22: Unknown column 'b' in 'where clause' +CALL sp; +ERROR 42S22: Unknown column 'b' in 'where clause' +DROP PROCEDURE sp; +DROP VIEW v1; +DROP TABLE t1, t2; # End of 5.5 test diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index cb93cd31442..fb9da936fdb 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9373,5 +9373,46 @@ CALL sp; DROP PROCEDURE sp; DROP TABLE t1; +--echo # +--echo # MDEV-17055: Server crashes in find_order_in_list upon +--echo # 2nd (3rd) execution of SP with UPDATE +--echo # + +CREATE TABLE t1 (a INT); +CREATE VIEW v1 AS SELECT * FROM t1; +CREATE TABLE t2 (c INT); + +CREATE PROCEDURE sp() UPDATE v1 SET a = 1 ORDER BY a, b LIMIT 1; +LOCK TABLE t2 READ; +--error ER_TABLE_NOT_LOCKED +CALL sp; +UNLOCK TABLES; +--error ER_BAD_FIELD_ERROR +CALL sp; +--error ER_BAD_FIELD_ERROR +CALL sp; +--error ER_BAD_FIELD_ERROR +CALL sp; + +# Cleanup +DROP PROCEDURE sp; + +CREATE PROCEDURE sp() UPDATE v1 SET a = 1 WHERE a=1 and b=2; +LOCK TABLE t2 READ; +--error ER_TABLE_NOT_LOCKED +CALL sp; +UNLOCK TABLES; +--error ER_BAD_FIELD_ERROR +CALL sp; +--error ER_BAD_FIELD_ERROR +CALL sp; +--error ER_BAD_FIELD_ERROR +CALL sp; + +# Cleanup +DROP PROCEDURE sp; + +DROP VIEW v1; +DROP TABLE t1, t2; --echo # End of 5.5 test diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index e990eba8e54..1709669d8a2 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -90,6 +90,7 @@ mysql_handle_derived(LEX *lex, uint phases) sl= sl->next_select_in_list()) { TABLE_LIST *cursor= sl->get_table_list(); + sl->changed_elements|= TOUCHED_SEL_DERIVED; /* DT_MERGE_FOR_INSERT is not needed for views/derived tables inside subqueries. Views and derived tables of subqueries should be @@ -1002,8 +1003,7 @@ bool mysql_derived_reinit(THD *thd, LEX *lex, TABLE_LIST *derived) derived->get_unit())); st_select_lex_unit *unit= derived->get_unit(); - if (derived->table) - derived->merged_for_insert= FALSE; + derived->merged_for_insert= FALSE; unit->unclean(); unit->types.empty(); /* for derived tables & PS (which can't be reset by Item_subquery) */ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index ba8134bfaf1..cc040508842 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1510,7 +1510,6 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, DBUG_RETURN(TRUE); } select_lex->fix_prepare_information(thd, &fake_conds, &fake_conds); - select_lex->first_execution= 0; } /* Only call prepare_for_posistion() if we are not performing a DELAYED diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 3c0d3049ae4..7d694f90aa2 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1904,7 +1904,7 @@ void st_select_lex::init_query() n_child_sum_items= 0; subquery_in_having= explicit_limit= 0; is_item_list_lookup= 0; - first_execution= 1; + changed_elements= 0; first_natural_join_processing= 1; first_cond_optimization= 1; parsing_place= NO_MATTER; @@ -3367,9 +3367,10 @@ void st_select_lex::fix_prepare_information(THD *thd, Item **conds, Item **having_conds) { DBUG_ENTER("st_select_lex::fix_prepare_information"); - if (!thd->stmt_arena->is_conventional() && first_execution) + if (!thd->stmt_arena->is_conventional() && + !(changed_elements & TOUCHED_SEL_COND)) { - first_execution= 0; + changed_elements|= TOUCHED_SEL_COND; if (group_list.first) { if (!group_list_ptrs) diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 8a31560d2bf..5b589714e1a 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -729,6 +729,10 @@ public: typedef class st_select_lex_unit SELECT_LEX_UNIT; + +#define TOUCHED_SEL_COND 1/* WHERE/HAVING/ON should be reinited before use */ +#define TOUCHED_SEL_DERIVED (1<<1)/* derived should be reinited before use */ + /* SELECT_LEX - store information of parsed SELECT statment */ @@ -876,7 +880,8 @@ public: subquery. Prepared statements work OK in that regard, as in case of an error during prepare the PS is not created. */ - bool first_execution; + uint8 changed_elements; // see TOUCHED_SEL_* + /* TODO: add foloowing first_* to bitmap above */ bool first_natural_join_processing; bool first_cond_optimization; /* do not wrap view fields with Item_ref */ diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 6aa6aacc504..16923ee5a21 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -2496,7 +2496,7 @@ void reinit_stmt_before_use(THD *thd, LEX *lex) } for (; sl; sl= sl->next_select_in_list()) { - if (!sl->first_execution) + if (sl->changed_elements & TOUCHED_SEL_COND) { /* remove option which was put by mysql_explain_union() */ sl->options&= ~SELECT_DESCRIBE; @@ -2543,19 +2543,28 @@ void reinit_stmt_before_use(THD *thd, LEX *lex) order->next= sl->group_list_ptrs->at(ix+1); } } + } + { // no harm to do it (item_ptr set on parsing) + ORDER *order; for (order= sl->group_list.first; order; order= order->next) + { order->item= &order->item_ptr; + } /* Fix ORDER list */ for (order= sl->order_list.first; order; order= order->next) - order->item= &order->item_ptr; { -#ifndef DBUG_OFF - bool res= -#endif - sl->handle_derived(lex, DT_REINIT); - DBUG_ASSERT(res == 0); + order->item= &order->item_ptr; } } + if (sl->changed_elements & TOUCHED_SEL_DERIVED) + { +#ifndef DBUG_OFF + bool res= +#endif + sl->handle_derived(lex, DT_REINIT); + DBUG_ASSERT(res == 0); + } + { SELECT_LEX_UNIT *unit= sl->master_unit(); unit->unclean(); diff --git a/sql/sql_union.cc b/sql/sql_union.cc index bbb4133417e..e855b64e600 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -215,8 +215,9 @@ st_select_lex_unit::init_prepare_fake_select_lex(THD *thd_arg, called at the first execution of the statement, while first_execution shows whether this is called at the first execution of the union that may form just a subselect. - */ - if (!fake_select_lex->first_execution && first_execution) + */ + if ((fake_select_lex->changed_elements & TOUCHED_SEL_COND) && + first_execution) { for (ORDER *order= global_parameters->order_list.first; order; From e39d6e0c53bccaf0c6b2a0341f8deb420f5e79be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 28 Feb 2019 23:11:15 +0200 Subject: [PATCH 28/44] MDEV-18601 Can't create table with ENCRYPTED=DEFAULT when innodb_default_encryption_key_id!=1 The problem with the InnoDB table attribute encryption_key_id is that it is not being persisted anywhere in InnoDB except if the table attribute encryption is specified and is something else than encryption=default. MDEV-17320 made it a hard error if encryption_key_id is specified to be anything else than 1 in that case. Ideally, we would always persist encryption_key_id in InnoDB. But, then we would have to be prepared for the case that when encryption is being enabled for a table whose encryption_key_id attribute refers to a non-existing key. In MariaDB Server 10.1, our best option remains to not store anything inside InnoDB. But, instead of returning the error that MDEV-17320 introduced, we should merely issue a warning that the specified encryption_key_id is going to be ignored if encryption=default. To improve the situation a little more, we will issue a warning if SET [GLOBAL|SESSION] innodb_default_encryption_key_id is being set to something that does not refer to an available encryption key. Starting with MariaDB Server 10.2, thanks to MDEV-5800, we could open the table definition from InnoDB side when the encryption is being enabled, and actually fix the root cause of what was reported in MDEV-17320. --- .../r/innodb-checksum-algorithm,32k.rdiff | 18 +-- .../r/innodb-checksum-algorithm,64k.rdiff | 18 +-- .../r/innodb-checksum-algorithm.result | 18 +++ .../r/innodb-compressed-blob.result | 2 + .../r/innodb-encryption-alter.result | 35 +++-- .../encryption/t/innodb-encryption-alter.test | 14 +- storage/innobase/handler/ha_innodb.cc | 120 +++++++++--------- storage/xtradb/handler/ha_innodb.cc | 119 +++++++++-------- 8 files changed, 182 insertions(+), 162 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-checksum-algorithm,32k.rdiff b/mysql-test/suite/encryption/r/innodb-checksum-algorithm,32k.rdiff index cd66df7440b..d963cde132a 100644 --- a/mysql-test/suite/encryption/r/innodb-checksum-algorithm,32k.rdiff +++ b/mysql-test/suite/encryption/r/innodb-checksum-algorithm,32k.rdiff @@ -1,5 +1,5 @@ --- suite/encryption/r/innodb-checksum-algorithm.result -+++ suite/encryption/r/innodb-checksum-algorithm,32k.reject ++++ suite/encryption/r/innodb-checksum-algorithm.result @@ -13,9 +13,9 @@ SET GLOBAL innodb_default_encryption_key_id=4; SET GLOBAL innodb_checksum_algorithm=crc32; @@ -9,10 +9,10 @@ create table tc_crc32(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_crc32(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_crc32(a serial, b blob, index(b(10))) engine=innodb -@@ -222,9 +222,9 @@ +@@ -153,9 +153,9 @@ t_crc32, tpe_crc32, tp_crc32; SET GLOBAL innodb_checksum_algorithm=innodb; create table tce_innodb(a serial, b blob, index(b(10))) engine=innodb @@ -21,10 +21,10 @@ create table tc_innodb(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_innodb(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_innodb(a serial, b blob, index(b(10))) engine=innodb -@@ -431,9 +431,9 @@ +@@ -293,9 +293,9 @@ t_innodb, tpe_innodb, tp_innodb; SET GLOBAL innodb_checksum_algorithm=none; create table tce_none(a serial, b blob, index(b(10))) engine=innodb @@ -33,6 +33,6 @@ create table tc_none(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_none(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_none(a serial, b blob, index(b(10))) engine=innodb diff --git a/mysql-test/suite/encryption/r/innodb-checksum-algorithm,64k.rdiff b/mysql-test/suite/encryption/r/innodb-checksum-algorithm,64k.rdiff index 523074297da..d963cde132a 100644 --- a/mysql-test/suite/encryption/r/innodb-checksum-algorithm,64k.rdiff +++ b/mysql-test/suite/encryption/r/innodb-checksum-algorithm,64k.rdiff @@ -1,5 +1,5 @@ --- suite/encryption/r/innodb-checksum-algorithm.result -+++ suite/encryption/r/innodb-checksum-algorithm,64k.reject ++++ suite/encryption/r/innodb-checksum-algorithm.result @@ -13,9 +13,9 @@ SET GLOBAL innodb_default_encryption_key_id=4; SET GLOBAL innodb_checksum_algorithm=crc32; @@ -9,10 +9,10 @@ create table tc_crc32(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_crc32(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_crc32(a serial, b blob, index(b(10))) engine=innodb -@@ -222,9 +222,9 @@ +@@ -153,9 +153,9 @@ t_crc32, tpe_crc32, tp_crc32; SET GLOBAL innodb_checksum_algorithm=innodb; create table tce_innodb(a serial, b blob, index(b(10))) engine=innodb @@ -21,10 +21,10 @@ create table tc_innodb(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_innodb(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_innodb(a serial, b blob, index(b(10))) engine=innodb -@@ -431,9 +431,9 @@ +@@ -293,9 +293,9 @@ t_innodb, tpe_innodb, tp_innodb; SET GLOBAL innodb_checksum_algorithm=none; create table tce_none(a serial, b blob, index(b(10))) engine=innodb @@ -33,6 +33,6 @@ create table tc_none(a serial, b blob, index(b(10))) engine=innodb -ROW_FORMAT=COMPRESSED encrypted=no; +ROW_FORMAT=DYNAMIC encrypted=no; + Warnings: + Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_none(a serial, b blob, index(b(10))) engine=innodb - encrypted=yes; - create table t_none(a serial, b blob, index(b(10))) engine=innodb diff --git a/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result b/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result index 7fdea4b67b5..6fa2e7c594f 100644 --- a/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result +++ b/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result @@ -16,14 +16,20 @@ create table tce_crc32(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=yes; create table tc_crc32(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_crc32(a serial, b blob, index(b(10))) engine=innodb encrypted=yes; create table t_crc32(a serial, b blob, index(b(10))) engine=innodb encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table tpe_crc32(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=yes; create table tp_crc32(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 begin; insert into tce_crc32(b) values (repeat('secret',20)); insert into tc_crc32(b) values (repeat('secret',20)); @@ -150,14 +156,20 @@ create table tce_innodb(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=yes; create table tc_innodb(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_innodb(a serial, b blob, index(b(10))) engine=innodb encrypted=yes; create table t_innodb(a serial, b blob, index(b(10))) engine=innodb encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table tpe_innodb(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=yes; create table tp_innodb(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 begin; insert into tce_innodb(b) values (repeat('secret',20)); insert into tc_innodb(b) values (repeat('secret',20)); @@ -284,14 +296,20 @@ create table tce_none(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=yes; create table tc_none(a serial, b blob, index(b(10))) engine=innodb ROW_FORMAT=COMPRESSED encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table te_none(a serial, b blob, index(b(10))) engine=innodb encrypted=yes; create table t_none(a serial, b blob, index(b(10))) engine=innodb encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 create table tpe_none(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=yes; create table tp_none(a serial, b blob, index(b(10))) engine=innodb page_compressed=yes encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 begin; insert into tce_none(b) values (repeat('secret',20)); insert into tc_none(b) values (repeat('secret',20)); diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index ce73b80820f..4187877be49 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -7,6 +7,8 @@ set GLOBAL innodb_default_encryption_key_id=4; create table t1(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed; create table t2(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes; create table t3(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=no; +Warnings: +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 insert into t1 values (1, repeat('secret',6000)); insert into t2 values (1, repeat('secret',6000)); insert into t3 values (1, repeat('secret',6000)); diff --git a/mysql-test/suite/encryption/r/innodb-encryption-alter.result b/mysql-test/suite/encryption/r/innodb-encryption-alter.result index 5245d1da7d0..b2f33b2b24d 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-alter.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-alter.result @@ -4,9 +4,16 @@ SET GLOBAL innodb_encrypt_tables = ON; SET GLOBAL innodb_encryption_threads = 4; CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO ENCRYPTION_KEY_ID=4; Warnings: -Warning 140 InnoDB: Ignored ENCRYPTION_KEY_ID 4 when encryption is disabled +Warning 140 InnoDB: ENCRYPTED=NO implies ENCRYPTION_KEY_ID=1 DROP TABLE t1; +set @save_global = @@GLOBAL.innodb_default_encryption_key_id; set innodb_default_encryption_key_id = 99; +Warnings: +Warning 1210 innodb_default_encryption_key=99 is not available +set global innodb_default_encryption_key_id = 99; +Warnings: +Warning 1210 innodb_default_encryption_key=99 is not available +set global innodb_default_encryption_key_id = @save_global; CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") SHOW WARNINGS; @@ -40,8 +47,6 @@ t1 CREATE TABLE `t1` ( PRIMARY KEY (`pk`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 `ENCRYPTION_KEY_ID`=4 CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO ENCRYPTION_KEY_ID=1; -Warnings: -Warning 140 InnoDB: Ignored ENCRYPTION_KEY_ID 1 when encryption is disabled ALTER TABLE t1 ENCRYPTION_KEY_ID=99; ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID' SHOW WARNINGS; @@ -53,37 +58,29 @@ drop table t1,t2; SET GLOBAL innodb_encrypt_tables=OFF; CREATE TABLE t1 (a int not null primary key) engine=innodb; ALTER TABLE t1 ENCRYPTION_KEY_ID=4; -ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID' -SHOW WARNINGS; -Level Code Message -Warning 140 InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1 -Error 1478 Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID' SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) NOT NULL, PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `ENCRYPTION_KEY_ID`=4 DROP TABLE t1; CREATE TABLE t2 (a int not null primary key) engine=innodb; ALTER TABLE t2 ENCRYPTION_KEY_ID=4, ALGORITHM=COPY; -ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options") -SHOW WARNINGS; -Level Code Message -Warning 140 InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1 -Error 1005 Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options") -Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( `a` int(11) NOT NULL, PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `ENCRYPTION_KEY_ID`=4 DROP TABLE t2; CREATE TABLE t3 (a int not null primary key) engine=innodb ENCRYPTION_KEY_ID=4; -ERROR HY000: Can't create table `test`.`t3` (errno: 140 "Wrong create options") +DROP TABLE t3; +SET GLOBAL innodb_encrypt_tables='FORCE'; +CREATE TABLE t1 (a int primary key) engine=innodb encrypted=no; +ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") SHOW WARNINGS; Level Code Message -Warning 140 InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1 -Error 1005 Can't create table `test`.`t3` (errno: 140 "Wrong create options") +Warning 140 InnoDB: ENCRYPTED=NO cannot be used with innodb_encrypt_tables=FORCE +Error 1005 Can't create table `test`.`t1` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB diff --git a/mysql-test/suite/encryption/t/innodb-encryption-alter.test b/mysql-test/suite/encryption/t/innodb-encryption-alter.test index 9465226dd96..711beeef1e1 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-alter.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-alter.test @@ -19,7 +19,10 @@ SET GLOBAL innodb_encryption_threads = 4; CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO ENCRYPTION_KEY_ID=4; DROP TABLE t1; +set @save_global = @@GLOBAL.innodb_default_encryption_key_id; set innodb_default_encryption_key_id = 99; +set global innodb_default_encryption_key_id = 99; +set global innodb_default_encryption_key_id = @save_global; --error 1005 CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB; SHOW WARNINGS; @@ -90,25 +93,26 @@ drop table t1,t2; # # MDEV-17230: encryption_key_id from alter is ignored by encryption threads # +--enable_warnings SET GLOBAL innodb_encrypt_tables=OFF; CREATE TABLE t1 (a int not null primary key) engine=innodb; ---error ER_ILLEGAL_HA_CREATE_OPTION ALTER TABLE t1 ENCRYPTION_KEY_ID=4; -SHOW WARNINGS; SHOW CREATE TABLE t1; DROP TABLE t1; CREATE TABLE t2 (a int not null primary key) engine=innodb; --replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/ ---error ER_CANT_CREATE_TABLE ALTER TABLE t2 ENCRYPTION_KEY_ID=4, ALGORITHM=COPY; --replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/ -SHOW WARNINGS; SHOW CREATE TABLE t2; DROP TABLE t2; ---error ER_CANT_CREATE_TABLE CREATE TABLE t3 (a int not null primary key) engine=innodb ENCRYPTION_KEY_ID=4; +DROP TABLE t3; + +SET GLOBAL innodb_encrypt_tables='FORCE'; +--error ER_CANT_CREATE_TABLE +CREATE TABLE t1 (a int primary key) engine=innodb encrypted=no; SHOW WARNINGS; # reset system diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 80ff2e02e13..c670839b5cd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2018, MariaDB Corporation. +Copyright (c) 2013, 2019, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -686,9 +686,25 @@ static int mysql_tmpfile_path(const char *path, const char *prefix) static void innodb_remember_check_sysvar_funcs(); mysql_var_check_func check_sysvar_enum; +/** Update callback for SET [SESSION] innodb_default_encryption_key_id */ +static void +innodb_default_encryption_key_id_update(THD* thd, st_mysql_sys_var* var, + void* var_ptr, const void *save) +{ + uint key_id = *static_cast(save); + if (key_id != FIL_DEFAULT_ENCRYPTION_KEY + && !encryption_key_id_exists(key_id)) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_WRONG_ARGUMENTS, + "innodb_default_encryption_key=%u" + " is not available", key_id); + } + *static_cast(var_ptr) = key_id; +} + static MYSQL_THDVAR_UINT(default_encryption_key_id, PLUGIN_VAR_RQCMDARG, "Default encryption key id used for table encryption.", - NULL, NULL, + NULL, innodb_default_encryption_key_id_update, FIL_DEFAULT_ENCRYPTION_KEY, 1, UINT_MAX32, 0); /** @@ -10862,8 +10878,7 @@ create_table_def( const char* remote_path, /*!< in: Remote path or zero length-string */ ulint flags, /*!< in: table flags */ ulint flags2, /*!< in: table flags2 */ - fil_encryption_t mode, /*!< in: encryption mode */ - ulint key_id) /*!< in: encryption key_id */ + const ha_table_option_struct*options) { THD* thd = trx->mysql_thd; dict_table_t* table; @@ -11053,7 +11068,9 @@ err_col: fts_add_doc_id_column(table, heap); } - err = row_create_table_for_mysql(table, trx, false, mode, key_id); + err = row_create_table_for_mysql(table, trx, false, + fil_encryption_t(options->encryption), + options->encryption_key_id); mem_heap_free(heap); @@ -11887,21 +11904,47 @@ ha_innobase::check_table_options( enum row_type row_format = table->s->row_type; ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; - fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - if (encrypt != FIL_ENCRYPTION_DEFAULT && !use_tablespace) { + switch (options->encryption) { + case FIL_ENCRYPTION_OFF: + if (options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTED=NO implies" + " ENCRYPTION_KEY_ID=1"); + compile_time_assert(FIL_DEFAULT_ENCRYPTION_KEY == 1); + } + if (srv_encrypt_tables != 2) { + break; + } push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTED requires innodb_file_per_table"); + "InnoDB: ENCRYPTED=NO cannot be used with" + " innodb_encrypt_tables=FORCE"); return "ENCRYPTED"; - } + case FIL_ENCRYPTION_DEFAULT: + if (!srv_encrypt_tables) { + break; + } + /* fall through */ + case FIL_ENCRYPTION_ON: + if (!encryption_key_id_exists(options->encryption_key_id)) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTION_KEY_ID %u not available", + options->encryption_key_id); + return "ENCRYPTION_KEY_ID"; + } + } - if (encrypt == FIL_ENCRYPTION_OFF && srv_encrypt_tables == 2) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTED=OFF cannot be used when innodb_encrypt_tables=FORCE"); + if (!use_tablespace && options->encryption != FIL_ENCRYPTION_DEFAULT) { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTED requires" + " innodb_file_per_table"); return "ENCRYPTED"; } @@ -11977,46 +12020,6 @@ ha_innobase::check_table_options( } } - /* If encryption is set up make sure that used key_id is found */ - if (encrypt == FIL_ENCRYPTION_ON || - (encrypt == FIL_ENCRYPTION_DEFAULT && srv_encrypt_tables)) { - if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTION_KEY_ID %u not available", - (uint)options->encryption_key_id - ); - return "ENCRYPTION_KEY_ID"; - } - } - - /* Ignore nondefault key_id if encryption is set off */ - if (encrypt == FIL_ENCRYPTION_OFF && - options->encryption_key_id != THDVAR(thd, default_encryption_key_id)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: Ignored ENCRYPTION_KEY_ID %u when encryption is disabled", - (uint)options->encryption_key_id - ); - options->encryption_key_id = FIL_DEFAULT_ENCRYPTION_KEY; - } - - /* If default encryption is used and encryption is disabled, you may - not use nondefault encryption_key_id as it is not stored anywhere. */ - if (encrypt == FIL_ENCRYPTION_DEFAULT - && !srv_encrypt_tables - && options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) { - compile_time_assert(FIL_DEFAULT_ENCRYPTION_KEY == 1); - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1" - ); - return "ENCRYPTION_KEY_ID"; - } - /* Check atomic writes requirements */ if (awrites == ATOMIC_WRITES_ON || (awrites == ATOMIC_WRITES_DEFAULT && srv_use_atomic_writes)) { @@ -12074,10 +12077,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - /* Cache table options */ - ha_table_option_struct *options= form->s->option_struct; - fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - uint key_id = (uint)options->encryption_key_id; DBUG_ENTER("ha_innobase::create"); @@ -12100,7 +12099,7 @@ ha_innobase::create( /* Validate create options if innodb_strict_mode is set. */ if (create_options_are_invalid( - thd, form, create_info, use_tablespace)) { + thd, form, create_info, use_tablespace)) { DBUG_RETURN(HA_WRONG_CREATE_OPTION); } @@ -12170,7 +12169,8 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); error = create_table_def(trx, form, norm_name, temp_path, - remote_path, flags, flags2, encrypt, key_id); + remote_path, flags, flags2, + form->s->option_struct); if (error) { goto cleanup; } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 4ed9f644b45..d943a87ab78 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -5,7 +5,7 @@ Copyright (c) 2013, 2018, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2018, MariaDB Corporation. +Copyright (c) 2013, 2019, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -687,9 +687,25 @@ ib_cb_t innodb_api_cb[] = { static void innodb_remember_check_sysvar_funcs(); mysql_var_check_func check_sysvar_enum; +/** Update callback for SET [SESSION] innodb_default_encryption_key_id */ +static void +innodb_default_encryption_key_id_update(THD* thd, st_mysql_sys_var* var, + void* var_ptr, const void *save) +{ + uint key_id = *static_cast(save); + if (key_id != FIL_DEFAULT_ENCRYPTION_KEY + && !encryption_key_id_exists(key_id)) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_WRONG_ARGUMENTS, + "innodb_default_encryption_key=%u" + " is not available", key_id); + } + *static_cast(var_ptr) = key_id; +} + static MYSQL_THDVAR_UINT(default_encryption_key_id, PLUGIN_VAR_RQCMDARG, "Default encryption key id used for table encryption.", - NULL, NULL, + NULL, innodb_default_encryption_key_id_update, FIL_DEFAULT_ENCRYPTION_KEY, 1, UINT_MAX32, 0); /** @@ -11430,8 +11446,7 @@ create_table_def( const char* remote_path, /*!< in: Remote path or zero length-string */ ulint flags, /*!< in: table flags */ ulint flags2, /*!< in: table flags2 */ - fil_encryption_t mode, /*!< in: encryption mode */ - ulint key_id) /*!< in: encryption key_id */ + const ha_table_option_struct*options) { THD* thd = trx->mysql_thd; dict_table_t* table; @@ -11622,7 +11637,9 @@ err_col: fts_add_doc_id_column(table, heap); } - err = row_create_table_for_mysql(table, trx, false, mode, key_id); + err = row_create_table_for_mysql(table, trx, false, + fil_encryption_t(options->encryption), + options->encryption_key_id); mem_heap_free(heap); @@ -12453,21 +12470,47 @@ ha_innobase::check_table_options( enum row_type row_format = table->s->row_type; ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; - fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - if (encrypt != FIL_ENCRYPTION_DEFAULT && !use_tablespace) { + switch (options->encryption) { + case FIL_ENCRYPTION_OFF: + if (options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTED=NO implies" + " ENCRYPTION_KEY_ID=1"); + compile_time_assert(FIL_DEFAULT_ENCRYPTION_KEY == 1); + } + if (srv_encrypt_tables != 2) { + break; + } push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTED requires innodb_file_per_table"); + "InnoDB: ENCRYPTED=NO cannot be used with" + " innodb_encrypt_tables=FORCE"); return "ENCRYPTED"; + case FIL_ENCRYPTION_DEFAULT: + if (!srv_encrypt_tables) { + break; + } + /* fall through */ + case FIL_ENCRYPTION_ON: + if (!encryption_key_id_exists(options->encryption_key_id)) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTION_KEY_ID %u not available", + options->encryption_key_id); + return "ENCRYPTION_KEY_ID"; + } } - if (encrypt == FIL_ENCRYPTION_OFF && srv_encrypt_tables == 2) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTED=OFF cannot be used when innodb_encrypt_tables=FORCE"); + if (!use_tablespace && options->encryption != FIL_ENCRYPTION_DEFAULT) { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: ENCRYPTED requires" + " innodb_file_per_table"); return "ENCRYPTED"; } @@ -12543,47 +12586,6 @@ ha_innobase::check_table_options( } } - /* If encryption is set up make sure that used key_id is found */ - if (encrypt == FIL_ENCRYPTION_ON || - (encrypt == FIL_ENCRYPTION_DEFAULT && srv_encrypt_tables)) { - if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTION_KEY_ID %u not available", - (uint)options->encryption_key_id - ); - return "ENCRYPTION_KEY_ID"; - - } - } - - /* Ignore nondefault key_id if encryption is set off */ - if (encrypt == FIL_ENCRYPTION_OFF && - options->encryption_key_id != THDVAR(thd, default_encryption_key_id)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: Ignored ENCRYPTION_KEY_ID %u when encryption is disabled", - (uint)options->encryption_key_id - ); - options->encryption_key_id = FIL_DEFAULT_ENCRYPTION_KEY; - } - - /* If default encryption is used and encryption is disabled, you may - not use nondefault encryption_key_id as it is not stored anywhere. */ - if (encrypt == FIL_ENCRYPTION_DEFAULT - && !srv_encrypt_tables - && options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) { - compile_time_assert(FIL_DEFAULT_ENCRYPTION_KEY == 1); - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1" - ); - return "ENCRYPTION_KEY_ID"; - } - /* Check atomic writes requirements */ if (awrites == ATOMIC_WRITES_ON || (awrites == ATOMIC_WRITES_DEFAULT && srv_use_atomic_writes)) { @@ -12641,10 +12643,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - /* Cache table options */ - ha_table_option_struct *options= form->s->option_struct; - fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - uint key_id = (uint)options->encryption_key_id; DBUG_ENTER("ha_innobase::create"); @@ -12667,7 +12665,7 @@ ha_innobase::create( /* Validate create options if innodb_strict_mode is set. */ if (create_options_are_invalid( - thd, form, create_info, use_tablespace)) { + thd, form, create_info, use_tablespace)) { DBUG_RETURN(HA_WRONG_CREATE_OPTION); } @@ -12743,7 +12741,8 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); error = create_table_def(trx, form, norm_name, temp_path, - remote_path, flags, flags2, encrypt, key_id); + remote_path, flags, flags2, + form->s->option_struct); if (error) { goto cleanup; } From 2d34713294d5339d1459911a46fa20395443b3c7 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 28 Feb 2019 23:40:49 +0100 Subject: [PATCH 29/44] Increase the version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 163137684a2..0924461861a 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=38 +MYSQL_VERSION_PATCH=39 From 4a1c66290dc81244f0f68c24666382fb2dae8f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 1 Mar 2019 10:16:56 +0200 Subject: [PATCH 30/44] MDEV-18775 Fix ALTER TABLE error handling for DROP INDEX On an error (such as when an index cannot be dropped due to FOREIGN KEY constraints), the field dict_index_t::to_be_dropped was only being cleared in debug builds, even though the field is available and being used also in non-debug builds. This was a regression that was introduced by myself originally in MySQL 5.7.6 and later merged to MariaDB 10.2.2, in https://github.com/mysql/mysql-server/commit/d39898de8e0de21f64ce94cd4ea698675edfb447 An error manifested itself in the MariaDB Server 10.4 non-debug build, involving instant ADD or DROP column. Because an earlier failed ALTER TABLE operation incorrectly left the dict_index_t::to_be_dropped flag set, the column pointers of the index fields would fail to be adjusted for instant ADD or DROP column (MDEV-15562). The instant ADD COLUMN in MariaDB Server 10.3 is unlikely to be affected by a similar scenario, because dict_table_t::instant_add_column() in 10.3 is applying the transformations to all indexes, not skipping to-be-dropped ones. --- mysql-test/suite/innodb/r/alter_table.result | 15 ++++++++++++++ mysql-test/suite/innodb/t/alter_table.test | 21 ++++++++++++++++++++ storage/innobase/handler/handler0alter.cc | 2 -- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/alter_table.result b/mysql-test/suite/innodb/r/alter_table.result index 2fb09e2fe5c..e47bfb90152 100644 --- a/mysql-test/suite/innodb/r/alter_table.result +++ b/mysql-test/suite/innodb/r/alter_table.result @@ -38,3 +38,18 @@ Warnings: Warning 1105 ORDER BY ignored as there is a user-defined clustered index in the table 't1' DROP TABLE t1; SET sql_mode=DEFAULT; +# +# MDEV-18775 Server crashes in dict_table_t::instant_column +# upon ADD COLUMN +# +CREATE TABLE tx (pk INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t1 (pk INT, a INT, PRIMARY KEY (pk), KEY (a), FOREIGN KEY (a) REFERENCES tx (pk)) ENGINE=InnoDB; +SET FOREIGN_KEY_CHECKS=OFF; +ALTER TABLE t1 DROP a; +ERROR HY000: Cannot drop column 'a': needed in a foreign key constraint 'test/t1_ibfk_1' +SET FOREIGN_KEY_CHECKS=ON; +ALTER TABLE t1 ADD b INT; +ALTER TABLE t1 DROP a; +ERROR HY000: Cannot drop index 'a': needed in a foreign key constraint +ALTER TABLE t1 ADD c INT; +DROP TABLE t1, tx; diff --git a/mysql-test/suite/innodb/t/alter_table.test b/mysql-test/suite/innodb/t/alter_table.test index ece24eb45c7..d0943e7d407 100644 --- a/mysql-test/suite/innodb/t/alter_table.test +++ b/mysql-test/suite/innodb/t/alter_table.test @@ -38,3 +38,24 @@ ALTER TABLE t1 ORDER BY a; DROP TABLE t1; SET sql_mode=DEFAULT; + +--echo # +--echo # MDEV-18775 Server crashes in dict_table_t::instant_column +--echo # upon ADD COLUMN +--echo # + +CREATE TABLE tx (pk INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t1 (pk INT, a INT, PRIMARY KEY (pk), KEY (a), FOREIGN KEY (a) REFERENCES tx (pk)) ENGINE=InnoDB; + +SET FOREIGN_KEY_CHECKS=OFF; + +--error ER_FK_COLUMN_CANNOT_DROP +ALTER TABLE t1 DROP a; + +SET FOREIGN_KEY_CHECKS=ON; + +ALTER TABLE t1 ADD b INT; +--error ER_DROP_INDEX_FK +ALTER TABLE t1 DROP a; +ALTER TABLE t1 ADD c INT; +DROP TABLE t1, tx; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 47cc31ce9dc..c04c69ba8e2 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -5142,14 +5142,12 @@ error_handled: ut_ad(!user_table->drop_aborted); err_exit: -#ifdef UNIV_DEBUG /* Clear the to_be_dropped flag in the data dictionary cache. */ for (ulint i = 0; i < ctx->num_to_drop_index; i++) { DBUG_ASSERT(ctx->drop_index[i]->is_committed()); DBUG_ASSERT(ctx->drop_index[i]->to_be_dropped); ctx->drop_index[i]->to_be_dropped = 0; } -#endif /* UNIV_DEBUG */ row_mysql_unlock_data_dictionary(ctx->trx); From 000c6cfc610bc3a0d205e81ca20726122314fb35 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Feb 2019 12:47:14 +0100 Subject: [PATCH 31/44] debian: more robust control file hacking when deleting a package, delete up to the next empty line, instead of relying on a package description being of some fixed number of lines long --- debian/autobake-deb.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index ac2f1395dcd..f53439340a4 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -24,8 +24,8 @@ then sed -i -e '/Add support for verbose builds/,+2d' debian/rules # Don't include test suite package on Travis-CI to make the build time shorter - sed '/Package: mariadb-test-data/,+26d' -i debian/control - sed '/Package: mariadb-test/,+34d' -i debian/control + sed '/Package: mariadb-test-data/,/^$/d' -i debian/control + sed '/Package: mariadb-test/,/^$/d' -i debian/control fi @@ -47,7 +47,7 @@ fi if ! apt-cache madison libcrack2-dev | grep 'libcrack2-dev *| *2\.9' >/dev/null 2>&1 then sed '/libcrack2-dev/d' -i debian/control - sed '/Package: mariadb-plugin-cracklib/,+9d' -i debian/control + sed '/Package: mariadb-plugin-cracklib/,/^$/d' -i debian/control fi # If libpcre3-dev (>= 2:8.35-3.2~) is not available (before Debian Jessie or Ubuntu Wily) @@ -82,11 +82,11 @@ GCCVERSION=$(gcc -dumpfullversion -dumpversion | sed -e 's/\.\([0-9][0-9]\)/\1/g # x86 32 bit. if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] then - sed '/Package: mariadb-plugin-rocksdb/,+10d' -i debian/control + sed '/Package: mariadb-plugin-rocksdb/,/^$/d' -i debian/control fi if [[ $GCCVERSION -lt 40800 ]] then - sed '/Package: mariadb-plugin-aws-key-management-10.2/,+12d' -i debian/control + sed '/Package: mariadb-plugin-aws-key-management-10.2/,/^$/d' -i debian/control fi # Don't build cassandra package if thrift is not installed From 20043cf650f84dc7fcd5e2b73d1db0c5ce15e103 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Feb 2019 12:48:05 +0100 Subject: [PATCH 32/44] remove aws kms plugin from debian builds too --- debian/autobake-deb.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index f53439340a4..1a43c064468 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -84,10 +84,9 @@ if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] then sed '/Package: mariadb-plugin-rocksdb/,/^$/d' -i debian/control fi -if [[ $GCCVERSION -lt 40800 ]] -then - sed '/Package: mariadb-plugin-aws-key-management-10.2/,/^$/d' -i debian/control -fi + +# Always remove aws plugin, see -DNOT_FOR_DISTRIBUTION in CMakeLists.txt +sed '/Package: mariadb-plugin-aws-key-management-10.2/,/^$/d' -i debian/control # Don't build cassandra package if thrift is not installed if [[ ! -f /usr/local/include/thrift/Thrift.h && ! -f /usr/include/thrift/Thrift.h ]] From 8d47d9ed8852d4fbc1c8708e06eeb180e15587c6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Feb 2019 15:15:34 +0100 Subject: [PATCH 33/44] SSL test fixes * fix CRL tests to work * regenerate certificates to be at least 2048 bit (fixes buster and rhel8 in buildbot) * update generate-ssl-cert.sh to generate crl files * make all SSL tests to use certificates generated in generate-ssl-cert.sh, remove unused certificates Backport from 10.4 9c60535f8676 --- mysql-test/disabled.def | 3 - mysql-test/include/have_openssl.inc | 7 +- mysql-test/lib/generate-ssl-certs.sh | 37 ++- mysql-test/r/ssl-crl-revoked-crl.result | 1 - mysql-test/r/ssl.result | 4 +- mysql-test/r/ssl_cert_verify.result | 5 - mysql-test/r/ssl_crl.result | 24 +- mysql-test/r/ssl_crl_clients-valid.result | 24 -- mysql-test/r/ssl_crl_clients.result | 6 + mysql-test/r/ssl_crl_clients_valid.result | 16 -- mysql-test/r/ssl_crl_clrpath.result | 23 -- mysql-test/std_data/ca-cert-verify.pem | 20 -- mysql-test/std_data/cacert.pem | 110 ++++---- mysql-test/std_data/cakey.pem | 52 ++-- mysql-test/std_data/client-cert.crl | 12 + mysql-test/std_data/client-cert.pem | 102 ++++---- mysql-test/std_data/client-key.pem | 38 ++- mysql-test/std_data/crl-ca-cert.pem | 63 ----- mysql-test/std_data/crl-client-cert.pem | 62 ----- mysql-test/std_data/crl-client-key.pem | 15 -- mysql-test/std_data/crl-client-revoked.crl | 10 - mysql-test/std_data/crl-server-cert.pem | 62 ----- mysql-test/std_data/crl-server-key.pem | 15 -- mysql-test/std_data/crldir/ed1f42db.r0 | 12 + mysql-test/std_data/crldir/fc725416.r0 | 10 - mysql-test/std_data/galera-cert.pem | 26 -- mysql-test/std_data/galera-key.pem | 28 -- .../std_data/galera-upgrade-ca-cert.pem | 40 --- .../std_data/galera-upgrade-server-cert.pem | 20 -- .../std_data/galera-upgrade-server-key.pem | 28 -- .../std_data/server-cert-verify-fail.pem | 19 -- .../std_data/server-cert-verify-pass.pem | 19 -- mysql-test/std_data/server-cert.crl | 12 + mysql-test/std_data/server-cert.pem | 100 +++---- .../std_data/server-key-verify-fail.pem | 27 -- .../std_data/server-key-verify-pass.pem | 27 -- mysql-test/std_data/server-key.pem | 38 ++- mysql-test/std_data/server-new-cert.pem | 81 ++++++ mysql-test/std_data/server-new-key.pem | 27 ++ mysql-test/std_data/server8k-cert.pem | 246 +++++++++--------- mysql-test/std_data/server8k-key.pem | 194 +++++++------- mysql-test/std_data/serversan-cert.pem | 92 ++++--- mysql-test/std_data/serversan-key.pem | 40 ++- mysql-test/suite/galera/t/galera_ssl.cnf | 4 +- .../suite/galera/t/galera_ssl_compression.cnf | 4 +- .../suite/galera/t/galera_ssl_upgrade.cnf | 4 +- .../suite/galera/t/galera_ssl_upgrade.test | 2 +- ...alera_sst_mariabackup_encrypt_with_key.cnf | 4 +- mysql-test/t/ssl_cert_verify.test | 43 --- mysql-test/t/ssl_crl-master.opt | 4 - mysql-test/t/ssl_crl.combinations | 5 + mysql-test/t/ssl_crl.test | 15 +- mysql-test/t/ssl_crl_clients-master.opt | 4 - mysql-test/t/ssl_crl_clients.test | 31 +-- mysql-test/t/ssl_crl_clients_valid-master.opt | 4 - mysql-test/t/ssl_crl_clients_valid.test | 23 -- mysql-test/t/ssl_crl_clrpath-master.opt | 4 - mysql-test/t/ssl_crl_clrpath.test | 16 -- 58 files changed, 760 insertions(+), 1204 deletions(-) delete mode 100644 mysql-test/r/ssl-crl-revoked-crl.result delete mode 100644 mysql-test/r/ssl_cert_verify.result delete mode 100644 mysql-test/r/ssl_crl_clients-valid.result delete mode 100644 mysql-test/r/ssl_crl_clients_valid.result delete mode 100644 mysql-test/r/ssl_crl_clrpath.result delete mode 100644 mysql-test/std_data/ca-cert-verify.pem create mode 100644 mysql-test/std_data/client-cert.crl delete mode 100644 mysql-test/std_data/crl-ca-cert.pem delete mode 100644 mysql-test/std_data/crl-client-cert.pem delete mode 100644 mysql-test/std_data/crl-client-key.pem delete mode 100644 mysql-test/std_data/crl-client-revoked.crl delete mode 100644 mysql-test/std_data/crl-server-cert.pem delete mode 100644 mysql-test/std_data/crl-server-key.pem create mode 100644 mysql-test/std_data/crldir/ed1f42db.r0 delete mode 100644 mysql-test/std_data/crldir/fc725416.r0 delete mode 100644 mysql-test/std_data/galera-cert.pem delete mode 100644 mysql-test/std_data/galera-key.pem delete mode 100644 mysql-test/std_data/galera-upgrade-ca-cert.pem delete mode 100644 mysql-test/std_data/galera-upgrade-server-cert.pem delete mode 100644 mysql-test/std_data/galera-upgrade-server-key.pem delete mode 100644 mysql-test/std_data/server-cert-verify-fail.pem delete mode 100644 mysql-test/std_data/server-cert-verify-pass.pem create mode 100644 mysql-test/std_data/server-cert.crl delete mode 100644 mysql-test/std_data/server-key-verify-fail.pem delete mode 100644 mysql-test/std_data/server-key-verify-pass.pem create mode 100644 mysql-test/std_data/server-new-cert.pem create mode 100644 mysql-test/std_data/server-new-key.pem delete mode 100644 mysql-test/t/ssl_cert_verify.test delete mode 100644 mysql-test/t/ssl_crl-master.opt create mode 100644 mysql-test/t/ssl_crl.combinations delete mode 100644 mysql-test/t/ssl_crl_clients-master.opt delete mode 100644 mysql-test/t/ssl_crl_clients_valid-master.opt delete mode 100644 mysql-test/t/ssl_crl_clients_valid.test delete mode 100644 mysql-test/t/ssl_crl_clrpath-master.opt delete mode 100644 mysql-test/t/ssl_crl_clrpath.test diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index b6991cc1d37..93fff886791 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -14,9 +14,6 @@ events_time_zone : Test is not predictable as it depends on precise timi read_many_rows_innodb : Bug#11748886 2010-11-15 mattiasj report already exists mysql_embedded : Bug#12561297 2011-05-14 Anitha Dependent on PB2 changes - eventum#41836 #show_explain : Psergey: random timeout in range-checked-for-each record query. -ssl_crl_clients_valid : broken upstream -ssl_crl : broken upstream -ssl_crl_clrpath : broken upstream innodb-wl5522-debug-zip : broken upstream innodb_bug12902967 : broken upstream file_contents : MDEV-6526 these files are not installed anymore diff --git a/mysql-test/include/have_openssl.inc b/mysql-test/include/have_openssl.inc index c9260123f9d..ee51ee4527a 100644 --- a/mysql-test/include/have_openssl.inc +++ b/mysql-test/include/have_openssl.inc @@ -1,7 +1,4 @@ --- source include/have_ssl_communication.inc -let $crllen=`select length(trim(coalesce(@@ssl_crl, ''))) + length(trim(coalesce(@@ssl_crlpath, '')))`; -if (!$crllen) -{ +if (`SELECT count(*) = 0 FROM information_schema.GLOBAL_VARIABLES WHERE + VARIABLE_NAME = 'have_openssl' AND VARIABLE_VALUE = 'YES'`){ skip Needs OpenSSL; } - diff --git a/mysql-test/lib/generate-ssl-certs.sh b/mysql-test/lib/generate-ssl-certs.sh index 8f15ba9d521..0894be3a299 100755 --- a/mysql-test/lib/generate-ssl-certs.sh +++ b/mysql-test/lib/generate-ssl-certs.sh @@ -10,30 +10,49 @@ rm -rf demoCA mkdir demoCA demoCA/newcerts touch demoCA/index.txt echo 01 > demoCA/serial +echo 01 > demoCA/crlnumber # CA certificate, self-signed openssl req -x509 -newkey rsa:2048 -keyout cakey.pem -out cacert.pem -days 7300 -nodes -subj '/CN=cacert/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -text # server certificate signing request and private key. Note the very long subject (for MDEV-7859) -openssl req -newkey rsa:1024 -keyout server-key.pem -out demoCA/server-req.pem -days 7300 -nodes -subj '/CN=localhost/C=FI/ST=state or province within country, in other certificates in this file it is the same as L/L=location, usually an address but often ambiguously used/OU=organizational unit name, a division name within an organization/O=organization name, typically a company name' +openssl req -newkey rsa:2048 -keyout server-key.pem -out demoCA/server-req.pem -days 7300 -nodes -subj '/CN=localhost/C=FI/ST=state or province within country, in other certificates in this file it is the same as L/L=location, usually an address but often ambiguously used/OU=organizational unit name, a division name within an organization/O=organization name, typically a company name' # convert the key to yassl compatible format openssl rsa -in server-key.pem -out server-key.pem # sign the server certificate with CA certificate -openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -infiles demoCA/server-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -in demoCA/server-req.pem +# server certificate with different validity period (MDEV-7598) +openssl req -newkey rsa:2048 -keyout server-new-key.pem -out demoCA/server-new-req.pem -days 7301 -nodes -subj '/CN=server-new/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' +openssl rsa -in server-new-key.pem -out server-new-key.pem +openssl ca -keyfile cakey.pem -days 7301 -batch -cert cacert.pem -policy policy_anything -out server-new-cert.pem -in demoCA/server-new-req.pem + +# 8K cert openssl req -newkey rsa:8192 -keyout server8k-key.pem -out demoCA/server8k-req.pem -days 7300 -nodes -subj '/CN=server8k/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' openssl rsa -in server8k-key.pem -out server8k-key.pem -openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -infiles demoCA/server8k-req.pem - -openssl req -newkey rsa:1024 -keyout client-key.pem -out demoCA/client-req.pem -days 7300 -nodes -subj '/CN=client/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -openssl rsa -in client-key.pem -out client-key.pem -openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -in demoCA/server8k-req.pem # with SubjectAltName, only for OpenSSL 1.0.2+ cat > demoCA/sanext.conf < $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server ---source include/wait_until_disconnected.inc - ---exec echo "restart:" $ssl_verify_fail_path > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---error 1 ---exec $MYSQL --protocol=tcp --ssl-ca=$MYSQL_TEST_DIR/std_data/ca-cert-verify.pem --ssl-verify-server-cert -e "SHOW STATUS like 'Ssl_version'" - ---echo #T2: Host name (localhost) as common name in the server certificate, server certificate verification should pass. ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server ---source include/wait_until_disconnected.inc - ---exec echo "restart:" $ssl_verify_pass_path > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---replace_result TLSv1.3 TLS_VERSION TLSv1.2 TLS_VERSION TLSv1.1 TLS_VERSION TLSv1 TLS_VERSION ---exec $MYSQL --protocol=tcp --ssl-ca=$MYSQL_TEST_DIR/std_data/ca-cert-verify.pem --ssl-verify-server-cert -e "SHOW STATUS like 'Ssl_version'" - ---echo # restart server using restart ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server ---source include/wait_until_disconnected.inc - ---exec echo "restart: " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc diff --git a/mysql-test/t/ssl_crl-master.opt b/mysql-test/t/ssl_crl-master.opt deleted file mode 100644 index 8500f8cd6e7..00000000000 --- a/mysql-test/t/ssl_crl-master.opt +++ /dev/null @@ -1,4 +0,0 @@ ---ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem ---ssl-key=$MYSQL_TEST_DIR/std_data/crl-server-key.pem ---ssl-cert=$MYSQL_TEST_DIR/std_data/crl-server-cert.pem ---ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl diff --git a/mysql-test/t/ssl_crl.combinations b/mysql-test/t/ssl_crl.combinations new file mode 100644 index 00000000000..abeec480510 --- /dev/null +++ b/mysql-test/t/ssl_crl.combinations @@ -0,0 +1,5 @@ +[file] +ssl-crl=$MYSQL_TEST_DIR/std_data/client-cert.crl + +[path] +ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir diff --git a/mysql-test/t/ssl_crl.test b/mysql-test/t/ssl_crl.test index 65c14837e50..dc30a9b5934 100644 --- a/mysql-test/t/ssl_crl.test +++ b/mysql-test/t/ssl_crl.test @@ -2,15 +2,12 @@ -- source include/not_embedded.inc -- source include/have_openssl.inc ---echo # test --crl for the client : should connect ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-valid-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-valid-cert.pem test --ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl -e "SHOW VARIABLES like '%ssl%';" - ---echo # test --crlpath for the client : should connect ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-valid-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-valid-cert.pem --ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir test -e "SHOW VARIABLES like '%ssl%';" +--echo # try logging in with a certificate not in the server's --ssl-crl : should succeed +--replace_result TLSv1.3 TLS_VERSION TLSv1.2 TLS_VERSION TLSv1.1 TLS_VERSION TLSv1 TLS_VERSION +--exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/server-new-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/server-new-cert.pem test -e "SHOW STATUS LIKE 'Ssl_version'" --echo # try logging in with a certificate in the server's --ssl-crl : should fail ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR +# OpenSSL 1.1.1a correctly rejects the certificate, but the error message is wrong +--replace_result "ERROR 2013 (HY000): Lost connection to MySQL server at 'reading authorization packet', system error: 0" "ERROR 2026 (HY000): SSL connection error: sslv3 alert certificate revoked" --error 1 ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-cert.pem test -e "SHOW VARIABLES like '%ssl%';" +--exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem test -e "SHOW STATUS LIKE 'Ssl_version'" 2>&1 diff --git a/mysql-test/t/ssl_crl_clients-master.opt b/mysql-test/t/ssl_crl_clients-master.opt deleted file mode 100644 index fa885a61a0c..00000000000 --- a/mysql-test/t/ssl_crl_clients-master.opt +++ /dev/null @@ -1,4 +0,0 @@ ---ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem ---ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-key.pem ---ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-cert.pem ---ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl diff --git a/mysql-test/t/ssl_crl_clients.test b/mysql-test/t/ssl_crl_clients.test index 7c05f498fbe..fc954a2fc38 100644 --- a/mysql-test/t/ssl_crl_clients.test +++ b/mysql-test/t/ssl_crl_clients.test @@ -4,38 +4,33 @@ --echo # Test clients with and without CRL lists -let $ssl_base = --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-server-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-server-cert.pem; -let $ssl_crl = $ssl_base --ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl; -let $ssl_crlpath = $ssl_base --ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir; +let $ssl_base = --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem --ssl-verify-server-cert; +let $ssl_crl = $ssl_base --ssl-crl=$MYSQL_TEST_DIR/std_data/server-cert.crl; +let $ssl_crlpath = $ssl_base --ssl-crlpath=$MYSQL_TMP_DIR; +# See `openssl x509 -in server-cert.pem -noout -issuer_hash` +copy_file $MYSQL_TEST_DIR/std_data/server-cert.crl $MYSQL_TMP_DIR/ed1f42db.r0; --echo ############ Test mysql ############## --echo # Test mysql connecting to a server with a certificate revoked by -crl ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --error 1 ---exec $MYSQL $ssl_crl test -e "SHOW VARIABLES like '%ssl%';" +--exec $MYSQL $ssl_crl test -e "SHOW STATUS LIKE 'Ssl_version'" 2>&1 --echo # Test mysql connecting to a server with a certificate revoked by -crlpath ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --error 1 ---exec $MYSQL $ssl_crlpath test -e "SHOW VARIABLES like '%ssl%';" +--exec $MYSQL $ssl_crlpath test -e "SHOW STATUS LIKE 'Ssl_version'" 2>&1 --echo ############ Test mysqladmin ############## -let $admin_prefix = --no-defaults; let $admin_suffix = --default-character-set=latin1 -S $MASTER_MYSOCK -P $MASTER_MYPORT -u root --password= ping; --echo # Test mysqladmin connecting to a server with a certificate revoked by -crl ---disable_result_log ---replace_regex /.*mysqladmin.*: connect/mysqladmin: connect/ ---error 2 ---exec $MYSQLADMIN $admin_prefix $ssl_crl $admin_suffix 2>&1 ---enable_result_log +--replace_regex /.*mysqladmin.*:/mysqladmin:/ +--error 1 +--exec $MYSQLADMIN $ssl_crl $admin_suffix 2>&1 ---disable_result_log --echo # Test mysqladmin connecting to a server with a certificate revoked by -crlpath ---replace_regex /.*mysqladmin.*: connect/mysqladmin: connect/ ---error 2 ---exec $MYSQLADMIN $admin_prefix $ssl_crlpath $admin_suffix 2>&1 ---enable_result_log +--replace_regex /.*mysqladmin.*:/mysqladmin:/ +--error 1 +--exec $MYSQLADMIN $ssl_crlpath $admin_suffix 2>&1 diff --git a/mysql-test/t/ssl_crl_clients_valid-master.opt b/mysql-test/t/ssl_crl_clients_valid-master.opt deleted file mode 100644 index 258df564eba..00000000000 --- a/mysql-test/t/ssl_crl_clients_valid-master.opt +++ /dev/null @@ -1,4 +0,0 @@ ---ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem ---ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-valid-key.pem ---ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-valid-cert.pem ---ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl diff --git a/mysql-test/t/ssl_crl_clients_valid.test b/mysql-test/t/ssl_crl_clients_valid.test deleted file mode 100644 index f08fbf09397..00000000000 --- a/mysql-test/t/ssl_crl_clients_valid.test +++ /dev/null @@ -1,23 +0,0 @@ -# This test should work in embedded server after we fix mysqltest --- source include/not_embedded.inc --- source include/have_openssl.inc - ---echo # Test clients with and without CRL lists - -let $ssl_base = --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-server-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-server-cert.pem; -let $ssl_crl = $ssl_base --ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl; -let $ssl_crlpath = $ssl_base --ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir; - - ---echo ############ Test mysql ############## - ---echo # Test mysql connecting to a server with an empty crl ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---exec $MYSQL $ssl_crl test -e "SHOW VARIABLES like '%ssl%';" 2>&1 - ---echo ############ Test mysqladmin ############## -let $admin_prefix = --no-defaults; -let $admin_suffix = --default-character-set=latin1 -S $MASTER_MYSOCK -P $MASTER_MYPORT -u root --password= ping; - ---echo # Test mysqladmin connecting to a server with an empty crl ---exec $MYSQLADMIN $admin_prefix $ssl_crl $admin_suffix 2>&1 diff --git a/mysql-test/t/ssl_crl_clrpath-master.opt b/mysql-test/t/ssl_crl_clrpath-master.opt deleted file mode 100644 index b1f486a322b..00000000000 --- a/mysql-test/t/ssl_crl_clrpath-master.opt +++ /dev/null @@ -1,4 +0,0 @@ ---ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem ---ssl-key=$MYSQL_TEST_DIR/std_data/crl-server-key.pem ---ssl-cert=$MYSQL_TEST_DIR/std_data/crl-server-cert.pem ---ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir diff --git a/mysql-test/t/ssl_crl_clrpath.test b/mysql-test/t/ssl_crl_clrpath.test deleted file mode 100644 index 50d84ad175e..00000000000 --- a/mysql-test/t/ssl_crl_clrpath.test +++ /dev/null @@ -1,16 +0,0 @@ -# This test should work in embedded server after we fix mysqltest --- source include/not_embedded.inc --- source include/have_openssl.inc - ---echo # test --crl for the client : should connect ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-valid-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-valid-cert.pem test --ssl-crl=$MYSQL_TEST_DIR/std_data/crl-client-revoked.crl -e "SHOW VARIABLES like '%ssl%';" - ---echo # test --crlpath for the client : should connect ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-valid-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-valid-cert.pem --ssl-crlpath=$MYSQL_TEST_DIR/std_data/crldir test -e "SHOW VARIABLES like '%ssl%';" - ---echo # try logging in with a certificate in the server's --ssl-crlpath : should fail ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR ---error 1 ---exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/crl-ca-cert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/crl-client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/crl-client-cert.pem test -e "SHOW VARIABLES like '%ssl%';" From 5e2d2053d824fce83b035c49129c581d9de056df Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 11 Feb 2019 18:59:10 +0100 Subject: [PATCH 34/44] bugfix: set mysql_real_data_home_len correctly after mysql_real_data_home was updated in get_options() --- sql/mysqld.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f4fd6658171..599694b01ef 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4299,6 +4299,8 @@ static int init_common_variables() if (IS_SYSVAR_AUTOSIZE(&server_version_ptr)) set_server_version(server_version, sizeof(server_version)); + mysql_real_data_home_len= strlen(mysql_real_data_home); + if (!opt_abort) { if (IS_SYSVAR_AUTOSIZE(&server_version_ptr)) From 67937360090d9e0315064d36b20f202ec5045c5c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Feb 2019 16:06:47 +0100 Subject: [PATCH 35/44] .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 786c87767d7..2567c41278f 100644 --- a/.gitignore +++ b/.gitignore @@ -189,9 +189,10 @@ storage/myisam/myisampack storage/myisam/rt_test storage/myisam/sp_test storage/rocksdb/ldb +storage/rocksdb/myrocks_hotbackup storage/rocksdb/mysql_ldb -storage/rocksdb/sst_dump storage/rocksdb/rdb_source_revision.h +storage/rocksdb/sst_dump storage/tokudb/PerconaFT/buildheader/db.h storage/tokudb/PerconaFT/buildheader/make_tdb storage/tokudb/PerconaFT/buildheader/runcat.sh From 4ca2079142e92f2ce682d463f2e175339e5d79bc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 1 Mar 2019 13:23:34 -0500 Subject: [PATCH 36/44] MDEV-18486 Database crash on a table with indexed virtual column don't do anything special for stored generated columns in MyISAM repair code. add an assert that if there are virtual indexed columns, they _must_ be beyond the file->s->base.reclength boundary --- mysql-test/suite/vcol/r/vcol_keys_myisam.result | 8 ++++++++ mysql-test/suite/vcol/t/vcol_keys_myisam.test | 10 ++++++++++ storage/myisam/ha_myisam.cc | 10 +++++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index bd013da9b03..cde93889848 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -375,3 +375,11 @@ repair table t1 extended; Table Op Msg_type Msg_text test.t1 repair status OK drop table t1; +create table t1 ( id int primary key, +hexid varchar(10) generated always as (hex(id)) stored, +key (hexid)) engine=myisam; +insert into t1 (id) select 100; +select * from t1; +id hexid +100 64 +drop table t1; diff --git a/mysql-test/suite/vcol/t/vcol_keys_myisam.test b/mysql-test/suite/vcol/t/vcol_keys_myisam.test index 337d14c3074..3269979fc9e 100644 --- a/mysql-test/suite/vcol/t/vcol_keys_myisam.test +++ b/mysql-test/suite/vcol/t/vcol_keys_myisam.test @@ -262,3 +262,13 @@ create table t1 ( insert into t1 values (null, 0); repair table t1 extended; drop table t1; + +# +# MDEV-18486 Database crash on a table with indexed virtual column +# +create table t1 ( id int primary key, + hexid varchar(10) generated always as (hex(id)) stored, + key (hexid)) engine=myisam; +insert into t1 (id) select 100; +select * from t1; +drop table t1; diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index dedf232523b..80bfdda18de 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -934,14 +934,18 @@ void ha_myisam::setup_vcols_for_repair(HA_CHECK *param) ulong new_vreclength= file->s->vreclength; for (Field **vf= table->vfield; *vf; vf++) { - uint vf_end= (*vf)->offset(table->record[0]) + (*vf)->pack_length_in_rec(); - set_if_bigger(new_vreclength, vf_end); - indexed_vcols|= (*vf)->flags & PART_KEY_FLAG; + if (!(*vf)->stored_in_db()) + { + uint vf_end= (*vf)->offset(table->record[0]) + (*vf)->pack_length_in_rec(); + set_if_bigger(new_vreclength, vf_end); + indexed_vcols|= (*vf)->flags & PART_KEY_FLAG; + } } if (!indexed_vcols) return; file->s->vreclength= new_vreclength; } + DBUG_ASSERT(file->s->base.reclength < file->s->vreclength); param->fix_record= compute_vcols; table->use_all_columns(); table->vcol_set= &table->s->all_set; From 19df45a705a48f5c8c21af0814cb5be7d816977b Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 27 Feb 2019 13:14:31 +0400 Subject: [PATCH 37/44] MDEV-18333 Slow_queries count doesn't increase when slow_query_log is turned off --- mysql-test/r/log_slow.result | 37 +++++++++++++++++++++ mysql-test/t/log_slow.test | 40 +++++++++++++++++++++++ sql/log_event.cc | 2 +- sql/sql_admin.cc | 5 --- sql/sql_admin.h | 8 ++--- sql/sql_alter.cc | 4 --- sql/sql_alter.h | 2 +- sql/sql_cmd.h | 15 +++++++++ sql/sql_lex.cc | 14 ++++++++ sql/sql_parse.cc | 62 ++++++++++++++++++++++++++++-------- sql/sql_partition_admin.cc | 1 - 11 files changed, 161 insertions(+), 29 deletions(-) diff --git a/mysql-test/r/log_slow.result b/mysql-test/r/log_slow.result index 02e04cfbb2d..c50130a7b7a 100644 --- a/mysql-test/r/log_slow.result +++ b/mysql-test/r/log_slow.result @@ -76,3 +76,40 @@ set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; truncate mysql.slow_log; +# +# MDEV-18333 Slow_queries count doesn't increase when slow_query_log is turned off +# +SET SESSION slow_query_log=OFF; +SET GLOBAL slow_query_log=OFF; +SET long_query_time=0.1; +# Although this query is disallowed by slow_query_log, it should still increment Slow_queries +SELECT VARIABLE_VALUE INTO @global_slow_queries +FROM INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE VARIABLE_NAME='SLOW_QUERIES'; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT +CAST(VARIABLE_VALUE AS UNSIGNED)-@global_slow_queries AS Slow_queries_increment +FROM +INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE +VARIABLE_NAME='SLOW_QUERIES'; +Slow_queries_increment +1 +# Although this query is disallowed by log_slow_filter, it should still increment Slow_queries +SET log_slow_filter=filesort; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT VARIABLE_VALUE INTO @global_slow_queries +FROM INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE VARIABLE_NAME='SLOW_QUERIES'; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT +CAST(VARIABLE_VALUE AS UNSIGNED)-@global_slow_queries AS Slow_queries_increment +FROM +INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE +VARIABLE_NAME='SLOW_QUERIES'; +Slow_queries_increment +1 +SET log_slow_filter=DEFAULT; +SET @@long_query_time=default; +SET GLOBAL slow_query_log= @org_slow_query_log; diff --git a/mysql-test/t/log_slow.test b/mysql-test/t/log_slow.test index 56e35bd5a20..edb1865b5ae 100644 --- a/mysql-test/t/log_slow.test +++ b/mysql-test/t/log_slow.test @@ -58,3 +58,43 @@ set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; truncate mysql.slow_log; + +--echo # +--echo # MDEV-18333 Slow_queries count doesn't increase when slow_query_log is turned off +--echo # + +SET SESSION slow_query_log=OFF; +SET GLOBAL slow_query_log=OFF; +SET long_query_time=0.1; + +--echo # Although this query is disallowed by slow_query_log, it should still increment Slow_queries + +SELECT VARIABLE_VALUE INTO @global_slow_queries + FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME='SLOW_QUERIES'; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT + CAST(VARIABLE_VALUE AS UNSIGNED)-@global_slow_queries AS Slow_queries_increment + FROM + INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE + VARIABLE_NAME='SLOW_QUERIES'; + +--echo # Although this query is disallowed by log_slow_filter, it should still increment Slow_queries + +SET log_slow_filter=filesort; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT VARIABLE_VALUE INTO @global_slow_queries + FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME='SLOW_QUERIES'; +SELECT sleep(0.2) INTO @tmp FROM DUAL; +SELECT + CAST(VARIABLE_VALUE AS UNSIGNED)-@global_slow_queries AS Slow_queries_increment + FROM + INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE + VARIABLE_NAME='SLOW_QUERIES'; +SET log_slow_filter=DEFAULT; + +SET @@long_query_time=default; +SET GLOBAL slow_query_log= @org_slow_query_log; diff --git a/sql/log_event.cc b/sql/log_event.cc index 1b372778b88..bb373a4ed84 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -4528,7 +4528,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, thd->variables.sql_log_slow= opt_log_slow_slave_statements; } - thd->enable_slow_log= thd->variables.sql_log_slow; + thd->enable_slow_log= true; mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); /* Finalize server status flags after executing a statement. */ thd->update_server_status(); diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index b8881f8f108..cbc9780d507 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -1246,7 +1246,6 @@ bool Sql_cmd_analyze_table::execute(THD *thd) FALSE, UINT_MAX, FALSE)) goto error; WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); - thd->enable_slow_log= opt_log_slow_admin_statements; res= mysql_admin_table(thd, first_table, &m_lex->check_opt, "analyze", lock_type, 1, 0, 0, 0, &handler::ha_analyze, 0); @@ -1278,8 +1277,6 @@ bool Sql_cmd_check_table::execute(THD *thd) if (check_table_access(thd, SELECT_ACL, first_table, TRUE, UINT_MAX, FALSE)) goto error; /* purecov: inspected */ - thd->enable_slow_log= opt_log_slow_admin_statements; - res= mysql_admin_table(thd, first_table, &m_lex->check_opt, "check", lock_type, 0, 0, HA_OPEN_FOR_REPAIR, 0, &handler::ha_check, &view_check); @@ -1303,7 +1300,6 @@ bool Sql_cmd_optimize_table::execute(THD *thd) FALSE, UINT_MAX, FALSE)) goto error; /* purecov: inspected */ WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); - thd->enable_slow_log= opt_log_slow_admin_statements; res= (specialflag & SPECIAL_NO_NEW_FUNC) ? mysql_recreate_table(thd, first_table, true) : mysql_admin_table(thd, first_table, &m_lex->check_opt, @@ -1336,7 +1332,6 @@ bool Sql_cmd_repair_table::execute(THD *thd) if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table, FALSE, UINT_MAX, FALSE)) goto error; /* purecov: inspected */ - thd->enable_slow_log= opt_log_slow_admin_statements; WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); res= mysql_admin_table(thd, first_table, &m_lex->check_opt, "repair", TL_WRITE, 1, diff --git a/sql/sql_admin.h b/sql/sql_admin.h index 96594fad0cb..ce7308434c5 100644 --- a/sql/sql_admin.h +++ b/sql/sql_admin.h @@ -28,7 +28,7 @@ int reassign_keycache_tables(THD* thd, KEY_CACHE *src_cache, /** Sql_cmd_analyze_table represents the ANALYZE TABLE statement. */ -class Sql_cmd_analyze_table : public Sql_cmd +class Sql_cmd_analyze_table : public Sql_cmd_admin { public: /** @@ -53,7 +53,7 @@ public: /** Sql_cmd_check_table represents the CHECK TABLE statement. */ -class Sql_cmd_check_table : public Sql_cmd +class Sql_cmd_check_table : public Sql_cmd_admin { public: /** @@ -77,7 +77,7 @@ public: /** Sql_cmd_optimize_table represents the OPTIMIZE TABLE statement. */ -class Sql_cmd_optimize_table : public Sql_cmd +class Sql_cmd_optimize_table : public Sql_cmd_admin { public: /** @@ -102,7 +102,7 @@ public: /** Sql_cmd_repair_table represents the REPAIR TABLE statement. */ -class Sql_cmd_repair_table : public Sql_cmd +class Sql_cmd_repair_table : public Sql_cmd_admin { public: /** diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 758489ddcfb..243cf4c790f 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -302,8 +302,6 @@ bool Sql_cmd_alter_table::execute(THD *thd) "INDEX DIRECTORY"); create_info.data_file_name= create_info.index_file_name= NULL; - thd->enable_slow_log= opt_log_slow_admin_statements; - #ifdef WITH_WSREP TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl); @@ -352,8 +350,6 @@ bool Sql_cmd_discard_import_tablespace::execute(THD *thd) if (check_grant(thd, ALTER_ACL, table_list, false, UINT_MAX, false)) return true; - thd->enable_slow_log= opt_log_slow_admin_statements; - /* Check if we attempt to alter mysql.slow_log or mysql.general_log table and return an error if diff --git a/sql/sql_alter.h b/sql/sql_alter.h index a4505f1d6c1..74dd8b0e5d8 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -363,7 +363,7 @@ private: statements. @todo move Alter_info and other ALTER generic structures from Lex here. */ -class Sql_cmd_common_alter_table : public Sql_cmd +class Sql_cmd_common_alter_table : public Sql_cmd_admin { protected: /** diff --git a/sql/sql_cmd.h b/sql/sql_cmd.h index 904578134b4..9cb2a728f41 100644 --- a/sql/sql_cmd.h +++ b/sql/sql_cmd.h @@ -145,6 +145,8 @@ public: */ virtual bool execute(THD *thd) = 0; + virtual bool log_slow_enabled_statement(const THD *thd) const; + protected: Sql_cmd() {} @@ -161,4 +163,17 @@ protected: } }; + +class Sql_cmd_admin: public Sql_cmd +{ +public: + Sql_cmd_admin() + {} + ~Sql_cmd_admin() + {} + bool log_slow_enabled_statement(const THD *thd) const; +}; + + + #endif // SQL_CMD_INCLUDED diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index c5aaa8e3cca..7c330151545 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -4719,6 +4719,20 @@ bool LEX::is_partition_management() const alter_info.flags == Alter_info::ALTER_REORGANIZE_PARTITION)); } + +bool Sql_cmd::log_slow_enabled_statement(const THD *thd) const +{ + return global_system_variables.sql_log_slow && thd->variables.sql_log_slow; +} + + +bool Sql_cmd_admin::log_slow_enabled_statement(const THD *thd) const +{ + return opt_log_slow_admin_statements && + Sql_cmd::log_slow_enabled_statement(thd); +} + + #ifdef MYSQL_SERVER uint binlog_unsafe_map[256]; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3f6ce8356ce..b6ad9fa2be3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1308,7 +1308,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, Commands which always take a long time are logged into the slow log only if opt_log_slow_admin_statements is set. */ - thd->enable_slow_log= thd->variables.sql_log_slow; + thd->enable_slow_log= true; thd->query_plan_flags= QPLAN_INIT; thd->lex->sql_command= SQLCOM_END; /* to avoid confusing VIEW detectors */ thd->reset_kill_query(); @@ -2013,6 +2013,31 @@ bool dispatch_command(enum enum_server_command command, THD *thd, } +static bool log_slow_enabled_statement(const THD *thd) +{ + /* + TODO-10.4: Add classes Sql_cmd_create_index and Sql_cmd_drop_index + for symmetry with other admin commands, so these statements can be + handled by this command: + */ + if (thd->lex->m_sql_cmd) + return thd->lex->m_sql_cmd->log_slow_enabled_statement(thd); + + /* + Currently CREATE INDEX or DROP INDEX cause a full table rebuild + and thus classify as slow administrative statements just like + ALTER TABLE. + */ + if ((thd->lex->sql_command == SQLCOM_CREATE_INDEX || + thd->lex->sql_command == SQLCOM_DROP_INDEX) && + !opt_log_slow_admin_statements) + return true; + + return global_system_variables.sql_log_slow && + thd->variables.sql_log_slow; +} + + /* @note This function must call delete_explain_query(). @@ -2029,13 +2054,18 @@ void log_slow_statement(THD *thd) if (unlikely(thd->in_sub_stmt)) goto end; // Don't set time for sub stmt + /* + Skip both long_query_count increment and logging if the current + statement forces slow log suppression (e.g. an SP statement). + + Note, we don't check for global_system_variables.sql_log_slow here. + According to the manual, the "Slow_queries" status variable does not require + sql_log_slow to be ON. So even if sql_log_slow is OFF, we still need to + continue and increment long_query_count (and skip only logging, see below): + */ + if (!thd->enable_slow_log) + goto end; // E.g. SP statement - /* Follow the slow log filter configuration. */ - if (!thd->enable_slow_log || !global_system_variables.sql_log_slow || - (thd->variables.log_slow_filter - && !(thd->variables.log_slow_filter & thd->query_plan_flags))) - goto end; - if (((thd->server_status & SERVER_QUERY_WAS_SLOW) || ((thd->server_status & (SERVER_QUERY_NO_INDEX_USED | SERVER_QUERY_NO_GOOD_INDEX_USED)) && @@ -2044,6 +2074,10 @@ void log_slow_statement(THD *thd) thd->get_examined_row_count() >= thd->variables.min_examined_row_limit) { thd->status_var.long_query_count++; + + if (!log_slow_enabled_statement(thd)) + goto end; + /* If rate limiting of slow log writes is enabled, decide whether to log this query to the log or not. @@ -2052,6 +2086,14 @@ void log_slow_statement(THD *thd) (global_query_id % thd->variables.log_slow_rate_limit) != 0) goto end; + /* + Follow the slow log filter configuration: + skip logging if the current statement matches the filter. + */ + if (thd->variables.log_slow_filter && + !(thd->variables.log_slow_filter & thd->query_plan_flags)) + goto end; + THD_STAGE_INFO(thd, stage_logging_slow_query); slow_log_print(thd, thd->query(), thd->query_length(), thd->utime_after_query); @@ -3527,12 +3569,6 @@ end_with_restore_list: if (check_one_table_access(thd, INDEX_ACL, all_tables)) goto error; /* purecov: inspected */ WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL) - /* - Currently CREATE INDEX or DROP INDEX cause a full table rebuild - and thus classify as slow administrative statements just like - ALTER TABLE. - */ - thd->enable_slow_log&= opt_log_slow_admin_statements; thd->query_plan_flags|= QPLAN_ADMIN; bzero((char*) &create_info, sizeof(create_info)); diff --git a/sql/sql_partition_admin.cc b/sql/sql_partition_admin.cc index 1ac9e748d42..d2fdee934ce 100644 --- a/sql/sql_partition_admin.cc +++ b/sql/sql_partition_admin.cc @@ -89,7 +89,6 @@ bool Sql_cmd_alter_table_exchange_partition::execute(THD *thd) /* Not allowed with EXCHANGE PARTITION */ DBUG_ASSERT(!create_info.data_file_name && !create_info.index_file_name); - thd->enable_slow_log= opt_log_slow_admin_statements; DBUG_RETURN(exchange_partition(thd, first_table, &alter_info)); } From 91e4f00389483d22fa81517bbcdcd21499fd283a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 4 Mar 2019 15:16:27 +0200 Subject: [PATCH 38/44] MDEV-18732 InnoDB: ALTER IGNORE returns error for NULL Only starting with MariaDB 10.3.8 (MDEV-16365), InnoDB can actually handle ALTER IGNORE TABLE correctly when introducing a NOT NULL attribute to a column that contains a NULL value. Between MariaDB Server 10.0 and 10.2, we would incorrectly return an error for ALTER IGNORE TABLE when the column contains a NULL value. --- .../innodb/r/innodb-alter-nullable.result | 34 +++++++++++++++++++ .../suite/innodb/t/innodb-alter-nullable.test | 26 ++++++++++++++ storage/innobase/handler/handler0alter.cc | 2 +- storage/xtradb/handler/handler0alter.cc | 2 +- 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-alter-nullable.result b/mysql-test/suite/innodb/r/innodb-alter-nullable.result index 2769bfdf1f5..3b96bac7a71 100644 --- a/mysql-test/suite/innodb/r/innodb-alter-nullable.result +++ b/mysql-test/suite/innodb/r/innodb-alter-nullable.result @@ -55,3 +55,37 @@ CREATE TABLE t1(c1 INT) ENGINE=InnoDB; ALTER TABLE t1 ADD CONSTRAINT UNIQUE KEY i1(c1); ALTER TABLE t1 CHANGE c1 c1 INT NOT NULL,ADD KEY(c1); DROP TABLE t1; +# +# MDEV-18732 InnoDB: ALTER IGNORE returns error for NULL +# +SET @mode = @@sql_mode; +SET sql_mode = STRICT_TRANS_TABLES; +CREATE TABLE t1(c INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (NULL); +ALTER IGNORE TABLE t1 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +INSERT INTO t1 VALUES (NULL); +ERROR 23000: Column 'c' cannot be null +SELECT * FROM t1; +c +0 +DROP TABLE t1; +CREATE TABLE t1(c INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (NULL),(1),(1); +ALTER IGNORE TABLE t1 ADD UNIQUE(c); +affected rows: 3 +info: Records: 3 Duplicates: 1 Warnings: 0 +ALTER IGNORE TABLE t1 ADD PRIMARY KEY(c); +affected rows: 2 +info: Records: 2 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +SELECT * FROM t1; +c +0 +1 +DROP TABLE t1; +SET sql_mode = @mode; diff --git a/mysql-test/suite/innodb/t/innodb-alter-nullable.test b/mysql-test/suite/innodb/t/innodb-alter-nullable.test index d039459f91f..81e464eddac 100644 --- a/mysql-test/suite/innodb/t/innodb-alter-nullable.test +++ b/mysql-test/suite/innodb/t/innodb-alter-nullable.test @@ -76,6 +76,32 @@ ALTER TABLE t1 ADD CONSTRAINT UNIQUE KEY i1(c1); ALTER TABLE t1 CHANGE c1 c1 INT NOT NULL,ADD KEY(c1); DROP TABLE t1; +--echo # +--echo # MDEV-18732 InnoDB: ALTER IGNORE returns error for NULL +--echo # + +SET @mode = @@sql_mode; +SET sql_mode = STRICT_TRANS_TABLES; +CREATE TABLE t1(c INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (NULL); +--enable_info +ALTER IGNORE TABLE t1 MODIFY c INT NOT NULL; +--disable_info +--error ER_BAD_NULL_ERROR +INSERT INTO t1 VALUES (NULL); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1(c INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (NULL),(1),(1); +--enable_info +ALTER IGNORE TABLE t1 ADD UNIQUE(c); +ALTER IGNORE TABLE t1 ADD PRIMARY KEY(c); +--disable_info +SELECT * FROM t1; +DROP TABLE t1; +SET sql_mode = @mode; + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index b3d9faf0680..2bcec2ce51f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -328,7 +328,7 @@ ha_innobase::check_if_supported_inplace_alter( NULL to a NOT NULL value. */ if ((ha_alter_info->handler_flags & Alter_inplace_info::ALTER_COLUMN_NOT_NULLABLE) - && !thd_is_strict_mode(user_thd)) { + && (ha_alter_info->ignore || !thd_is_strict_mode(user_thd))) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_NOT_NULL); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 43df8b04201..fa41c963464 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -332,7 +332,7 @@ ha_innobase::check_if_supported_inplace_alter( NULL to a NOT NULL value. */ if ((ha_alter_info->handler_flags & Alter_inplace_info::ALTER_COLUMN_NOT_NULLABLE) - && !thd_is_strict_mode(user_thd)) { + && (ha_alter_info->ignore || !thd_is_strict_mode(user_thd))) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_NOT_NULL); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); From 4fbf86454f62f1a7859ca4c6ef4fd3002a3c9881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 4 Mar 2019 16:21:16 +0200 Subject: [PATCH 39/44] Fix Galera results --- mysql-test/suite/galera/r/galera_mdl_race.result | 1 + mysql-test/suite/galera/r/galera_query_cache_sync_wait.result | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/r/galera_mdl_race.result b/mysql-test/suite/galera/r/galera_mdl_race.result index 639a84a286c..968258e8ec3 100644 --- a/mysql-test/suite/galera/r/galera_mdl_race.result +++ b/mysql-test/suite/galera/r/galera_mdl_race.result @@ -13,6 +13,7 @@ connection node_1; SET @@debug_dbug = "d,sync.wsrep_before_mdl_wait"; SELECT * FROM t2;; connection node_1a; +SET @@debug_dbug = "d,sync.wsrep_after_BF_victim_lock"; connection node_2; UPDATE t1 SET f2 = 'c' WHERE f1 = 1; connection node_1a; diff --git a/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result b/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result index 154f670cc79..a994852a78a 100644 --- a/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result +++ b/mysql-test/suite/galera/r/galera_query_cache_sync_wait.result @@ -41,8 +41,9 @@ connection node_1; INSERT INTO t1 VALUES (5); connection node_2; SELECT MAX(id) FROM t1 ; -SET GLOBAL debug_dbug = ""; +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; connection node_2a; +SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; connection node_2; MAX(id) From 74d648db12e100c628548fb2e5aa67de718bd1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 4 Mar 2019 16:25:14 +0200 Subject: [PATCH 40/44] Make a size_t-to-uint conversion explicit --- sql/mysqld.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 599694b01ef..1352bb64be8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2017, MariaDB Corporation. + Copyright (c) 2008, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4299,7 +4299,7 @@ static int init_common_variables() if (IS_SYSVAR_AUTOSIZE(&server_version_ptr)) set_server_version(server_version, sizeof(server_version)); - mysql_real_data_home_len= strlen(mysql_real_data_home); + mysql_real_data_home_len= uint(strlen(mysql_real_data_home)); if (!opt_abort) { From 730ed9907b2122d8ca2b7869fe0be4203d4ec59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 4 Mar 2019 18:38:42 +0200 Subject: [PATCH 41/44] After-merge fix: Add explicit type conversion Lesson learned: A HA_TOPTION_SYSVAR that is bound to MYSQL_THDVAR_UINT does not have the type uint, but ulonglong. --- storage/innobase/handler/ha_innodb.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e9e05810e81..9bd85142e9f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11301,7 +11301,7 @@ err_col: err = row_create_table_for_mysql( table, m_trx, fil_encryption_t(options->encryption), - options->encryption_key_id); + uint32_t(options->encryption_key_id)); m_drop_before_rollback = (err == DB_SUCCESS); } @@ -11800,12 +11800,13 @@ create_table_info_t::check_table_options() } /* fall through */ case FIL_ENCRYPTION_ON: - if (!encryption_key_id_exists(options->encryption_key_id)) { + const uint32_t key_id = uint32_t(options->encryption_key_id); + if (!encryption_key_id_exists(key_id)) { push_warning_printf( m_thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, "InnoDB: ENCRYPTION_KEY_ID %u not available", - options->encryption_key_id); + key_id); return "ENCRYPTION_KEY_ID"; } From 8f4de38f65ba89c6273c15c9adb50ab762d03f59 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 4 Mar 2019 23:10:30 -0800 Subject: [PATCH 42/44] MDEV-18467 Server crashes in fix_semijoin_strategies_for_picked_join_order If a splittable materialized derived table / view T is used in a inner nest of an outer join with impossible ON condition then T is marked as a constant table. Yet the execution plan to build T is still searched for in spite of the fact that is not needed. So it should be set. --- mysql-test/main/derived_split_innodb.result | 41 +++++++++++++++++++++ mysql-test/main/derived_split_innodb.test | 29 +++++++++++++++ sql/opt_split.cc | 13 ++++--- sql/sql_select.cc | 1 + sql/sql_select.h | 3 +- 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index 21dbd494e4b..b9ed016429b 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -99,3 +99,44 @@ id select_type table type possible_keys key key_len ref rows Extra 2 LATERAL DERIVED t1 eq_ref PRIMARY PRIMARY 4 test.t2.id 1 set join_cache_level=default; DROP TABLE t1,t2; +# +# Bug mdev-18467: join of grouping view and a base table as inner operand +# of left join with on condition containing impossible range +# +create table t1 (f1 int, f2 int, key(f2)) engine=InnoDB; +insert into t1 values (3,33), (7,77), (1,11); +create table t2 (f1 int, f2 int, primary key (f1)) engine=InnoDB; +insert into t2 values (3,33), (9,99), (1,11); +create view v1 as +select f1, max(f2) as f2 from t2 group by f1; +select t.f2 +from t1 +left join +(v1 join t1 as t on v1.f1=t.f1 and t.f2 = null) +on t1.f1=t.f1; +f2 +NULL +NULL +NULL +explain select t.f2 +from t1 +left join +(v1 join t1 as t on v1.f1=t.f1 and t.f2 = null) +on t1.f1=t.f1; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t const f2 NULL NULL NULL 1 Impossible ON condition +1 PRIMARY const key1 NULL NULL NULL 1 Impossible ON condition +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 +2 DERIVED t2 ALL PRIMARY NULL NULL NULL 3 Using temporary; Using filesort +set statement optimizer_switch='split_materialized=off' for explain select t.f2 +from t1 +left join +(v1 join t1 as t on v1.f1=t.f1 and t.f2 = null) +on t1.f1=t.f1; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t const f2 NULL NULL NULL 1 Impossible ON condition +1 PRIMARY const key1 NULL NULL NULL 1 Impossible ON condition +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 +2 DERIVED t2 index NULL PRIMARY 4 NULL 3 +drop view v1; +drop table t1,t2; diff --git a/mysql-test/main/derived_split_innodb.test b/mysql-test/main/derived_split_innodb.test index c3b3bcabede..1bf70cd8114 100644 --- a/mysql-test/main/derived_split_innodb.test +++ b/mysql-test/main/derived_split_innodb.test @@ -94,3 +94,32 @@ eval EXPLAIN $q; set join_cache_level=default; DROP TABLE t1,t2; + +--echo # +--echo # Bug mdev-18467: join of grouping view and a base table as inner operand +--echo # of left join with on condition containing impossible range +--echo # + +create table t1 (f1 int, f2 int, key(f2)) engine=InnoDB; +insert into t1 values (3,33), (7,77), (1,11); + +create table t2 (f1 int, f2 int, primary key (f1)) engine=InnoDB; +insert into t2 values (3,33), (9,99), (1,11); + +create view v1 as + select f1, max(f2) as f2 from t2 group by f1; + +let $q= +select t.f2 + from t1 + left join + (v1 join t1 as t on v1.f1=t.f1 and t.f2 = null) + on t1.f1=t.f1; + +eval $q; +eval explain $q; +eval set statement optimizer_switch='split_materialized=off' for explain $q; + +drop view v1; + +drop table t1,t2; diff --git a/sql/opt_split.cc b/sql/opt_split.cc index fc3f08464f4..cfac0c93544 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -1078,6 +1078,7 @@ bool JOIN::inject_best_splitting_cond(table_map remaining_tables) @param spl_plan info on the splitting plan chosen for the splittable table T remaining_tables the table T is joined just before these tables + is_const_table the table T is a constant table @details If in the final query plan the optimizer has chosen a splitting plan @@ -1091,12 +1092,13 @@ bool JOIN::inject_best_splitting_cond(table_map remaining_tables) */ bool JOIN_TAB::fix_splitting(SplM_plan_info *spl_plan, - table_map remaining_tables) + table_map remaining_tables, + bool is_const_table) { SplM_opt_info *spl_opt_info= table->spl_opt_info; DBUG_ASSERT(table->spl_opt_info != 0); JOIN *md_join= spl_opt_info->join; - if (spl_plan) + if (spl_plan && !is_const_table) { memcpy((char *) md_join->best_positions, (char *) spl_plan->best_positions, @@ -1113,7 +1115,7 @@ bool JOIN_TAB::fix_splitting(SplM_plan_info *spl_plan, remaining_tables, true); } - else + else if (md_join->save_qep) { md_join->restore_query_plan(md_join->save_qep); } @@ -1143,10 +1145,11 @@ bool JOIN::fix_all_splittings_in_plan() { POSITION *cur_pos= &best_positions[tablenr]; JOIN_TAB *tab= cur_pos->table; - if (tablenr >= const_tables && tab->table->is_splittable()) + if (tab->table->is_splittable()) { SplM_plan_info *spl_plan= cur_pos->spl_plan; - if (tab->fix_splitting(spl_plan, all_tables & ~prev_tables)) + if (tab->fix_splitting(spl_plan, all_tables & ~prev_tables, + tablenr < const_tables )) return true; } prev_tables|= tab->table->map; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 0bc27f18d47..e084ad74224 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -6689,6 +6689,7 @@ void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key) next=tmp; } join->best_ref[idx]=table; + join->positions[idx].spl_plan= 0; } diff --git a/sql/sql_select.h b/sql/sql_select.h index 0e486c1fbec..57d8dab8258 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -659,7 +659,8 @@ typedef struct st_join_table { void add_keyuses_for_splitting(); SplM_plan_info *choose_best_splitting(double record_count, table_map remaining_tables); - bool fix_splitting(SplM_plan_info *spl_plan, table_map remaining_tables); + bool fix_splitting(SplM_plan_info *spl_plan, table_map remaining_tables, + bool is_const_table); } JOIN_TAB; From 90f09ba8c249e23e015ce9d1d56463869f1a5358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Mar 2019 11:53:23 +0200 Subject: [PATCH 43/44] MDEV-18819: Add a test case --- .../innodb/r/innodb-alter-nullable.result | 116 ++++++++++++++++++ .../suite/innodb/t/innodb-alter-nullable.test | 68 ++++++++++ 2 files changed, 184 insertions(+) diff --git a/mysql-test/suite/innodb/r/innodb-alter-nullable.result b/mysql-test/suite/innodb/r/innodb-alter-nullable.result index 22aa39ba923..7f42c17027f 100644 --- a/mysql-test/suite/innodb/r/innodb-alter-nullable.result +++ b/mysql-test/suite/innodb/r/innodb-alter-nullable.result @@ -87,3 +87,119 @@ c 0 1 DROP TABLE t1; +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +SET @old_sql_mode = @@sql_mode; +SET sql_mode = ''; +ALTER TABLE t1 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +ALTER TABLE t2 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +ALTER TABLE t3 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +SET sql_mode = @old_sql_mode; +SELECT * FROM t1; +c g +0 0 +SELECT * FROM t2; +c v +0 0 +SELECT * FROM t3; +c v +0 0 +SELECT v FROM t3 FORCE INDEX(v); +v +0 +CHECK TABLE t1,t2,t3; +Table Op Msg_type Msg_text +test.t1 check status OK +test.t2 check status OK +test.t3 check status OK +DROP TABLE t1,t2,t3; +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +ALTER IGNORE TABLE t1 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +ALTER IGNORE TABLE t2 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +ALTER IGNORE TABLE t3 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 1 +Warnings: +Warning 1265 Data truncated for column 'c' at row 1 +SELECT * FROM t1; +c g +0 0 +SELECT * FROM t2; +c v +0 0 +SELECT * FROM t3; +c v +0 0 +SELECT v FROM t3 FORCE INDEX(v); +v +0 +CHECK TABLE t1,t2,t3; +Table Op Msg_type Msg_text +test.t1 check status OK +test.t2 check status OK +test.t3 check status OK +DROP TABLE t1,t2,t3; +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +ALTER TABLE t1 MODIFY c INT NOT NULL; +ERROR 22004: Invalid use of NULL value +ALTER TABLE t2 MODIFY c INT NOT NULL; +ERROR 22004: Invalid use of NULL value +ALTER TABLE t3 MODIFY c INT NOT NULL; +ERROR 01000: Data truncated for column 'c' at row 1 +UPDATE t1 SET c=0; +UPDATE t2 SET c=0; +UPDATE t3 SET c=0; +ALTER TABLE t1 MODIFY c INT NOT NULL; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +ALTER TABLE t2 MODIFY c INT NOT NULL; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +# MDEV-18819 FIXME: This should not require ALGORITHM=COPY. +ALTER TABLE t3 MODIFY c INT NOT NULL; +affected rows: 1 +info: Records: 1 Duplicates: 0 Warnings: 0 +SELECT * FROM t1; +c g +0 0 +SELECT * FROM t2; +c v +0 0 +SELECT * FROM t3; +c v +0 0 +DROP TABLE t1,t2,t3; diff --git a/mysql-test/suite/innodb/t/innodb-alter-nullable.test b/mysql-test/suite/innodb/t/innodb-alter-nullable.test index 19d3df75b5d..8a16d15066d 100644 --- a/mysql-test/suite/innodb/t/innodb-alter-nullable.test +++ b/mysql-test/suite/innodb/t/innodb-alter-nullable.test @@ -89,6 +89,74 @@ ALTER IGNORE TABLE t1 ADD PRIMARY KEY(c); SELECT * FROM t1; DROP TABLE t1; +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +SET @old_sql_mode = @@sql_mode; +# Allow lossy conversions of data +SET sql_mode = ''; +--enable_info +ALTER TABLE t1 MODIFY c INT NOT NULL; +ALTER TABLE t2 MODIFY c INT NOT NULL; +ALTER TABLE t3 MODIFY c INT NOT NULL; +--disable_info +SET sql_mode = @old_sql_mode; +SELECT * FROM t1; +SELECT * FROM t2; +SELECT * FROM t3; +SELECT v FROM t3 FORCE INDEX(v); +CHECK TABLE t1,t2,t3; +DROP TABLE t1,t2,t3; + +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +--enable_info +ALTER IGNORE TABLE t1 MODIFY c INT NOT NULL; +ALTER IGNORE TABLE t2 MODIFY c INT NOT NULL; +ALTER IGNORE TABLE t3 MODIFY c INT NOT NULL; +--disable_info +SELECT * FROM t1; +SELECT * FROM t2; +SELECT * FROM t3; +SELECT v FROM t3 FORCE INDEX(v); +CHECK TABLE t1,t2,t3; +DROP TABLE t1,t2,t3; + +CREATE TABLE t1(c INT, g INT AS (c) PERSISTENT) ENGINE=InnoDB; +CREATE TABLE t2(c INT, v INT AS (c) VIRTUAL) ENGINE=InnoDB; +CREATE TABLE t3(c INT, v INT AS (c) VIRTUAL, INDEX(v)) ENGINE=InnoDB; +INSERT INTO t1 SET c=NULL; +INSERT INTO t2 SET c=NULL; +INSERT INTO t3 SET c=NULL; +--enable_info +--error ER_INVALID_USE_OF_NULL +ALTER TABLE t1 MODIFY c INT NOT NULL; +--error ER_INVALID_USE_OF_NULL +ALTER TABLE t2 MODIFY c INT NOT NULL; +--error WARN_DATA_TRUNCATED +ALTER TABLE t3 MODIFY c INT NOT NULL; +--disable_info +UPDATE t1 SET c=0; +UPDATE t2 SET c=0; +UPDATE t3 SET c=0; +--enable_info +ALTER TABLE t1 MODIFY c INT NOT NULL; +ALTER TABLE t2 MODIFY c INT NOT NULL; +--echo # MDEV-18819 FIXME: This should not require ALGORITHM=COPY. +ALTER TABLE t3 MODIFY c INT NOT NULL; +--disable_info +SELECT * FROM t1; +SELECT * FROM t2; +SELECT * FROM t3; +DROP TABLE t1,t2,t3; + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc From 723ffdb32ee785cbc511abc457eb70d41c2fcce3 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 5 Mar 2019 14:11:42 +0100 Subject: [PATCH 44/44] MDEV-9519: After-merge fix for 10.3 --- sql/handler.cc | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index c41a8b5860e..53c58e64782 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3409,10 +3409,32 @@ int handler::update_auto_increment() if (unlikely(tmp)) // Out of range value in store { /* - It's better to return an error here than getting a confusing - 'duplicate key error' later. + First, test if the query was aborted due to strict mode constraints + or new field value greater than maximum integer value: */ - result= HA_ERR_AUTOINC_ERANGE; + if (thd->killed == KILL_BAD_DATA || + nr > table->next_number_field->get_max_int_value()) + { + /* + It's better to return an error here than getting a confusing + 'duplicate key error' later. + */ + result= HA_ERR_AUTOINC_ERANGE; + } + else + { + /* + Field refused this value (overflow) and truncated it, use the result + of the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong)nr, TRUE))) + nr= table->next_number_field->val_int(); + } } if (append) {