From 155a8a5215ee98051fb28a138d21009d59d25ce4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 3 Dec 2019 22:21:05 +0100 Subject: [PATCH 01/38] .clang-format - do *not* sort include files It is C++, not Java, the order of includes is often important. --- .clang-format | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-format b/.clang-format index 1ad93ead80a..725e4c67e02 100644 --- a/.clang-format +++ b/.clang-format @@ -90,7 +90,6 @@ PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right ReflowComments: true -SortIncludes: true SortUsingDeclarations: true SpaceAfterCStyleCast: true SpaceAfterLogicalNot: false From 05e72a33331bd9ad96ebbe4c844b94f0a5fb7390 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 3 Dec 2019 22:22:23 +0100 Subject: [PATCH 02/38] .clang-format - do not sort include files. It is C++, not Java, the order of includes is often important. --- .clang-format | 1 + 1 file changed, 1 insertion(+) diff --git a/.clang-format b/.clang-format index 725e4c67e02..5fc6ab23622 100644 --- a/.clang-format +++ b/.clang-format @@ -90,6 +90,7 @@ PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right ReflowComments: true +SortIncludes: false SortUsingDeclarations: true SpaceAfterCStyleCast: true SpaceAfterLogicalNot: false From c5dafca87e8684749500708c6b83e9e2dae5b429 Mon Sep 17 00:00:00 2001 From: Axel Schwenke Date: Wed, 4 Dec 2019 14:28:13 +0100 Subject: [PATCH 03/38] MDEV-17571 Make systemd timeout behavior more compatible with long Galera SSTs Set an explicit start and stop timeout of 900 seconds for the MariaDB Server systemd service --- support-files/mariadb.service.in | 7 +++++++ support-files/mariadb@.service.in | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index ef9fa5c2a22..ede97c04112 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -120,6 +120,13 @@ UMask=007 # LOAD DATA INFILE you can enable PrivateTmp=true for a little more security. PrivateTmp=false +# Set an explicit Start and Stop timeout of 900 seconds (15 minutes!) +# this is the same value as used in SysV init scripts in the past +# Galera might need a longer timeout, check the KB if you want to change this: +# https://mariadb.com/kb/en/library/systemd/#configuring-the-systemd-service-timeout +TimeoutStartSec=900 +TimeoutStopSec=900 + ## ## Options previously available to be set via [mysqld_safe] ## that now needs to be set by systemd config files as mysqld_safe diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 8afae3bd09b..1f84b582e4e 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -141,6 +141,13 @@ UMask=007 # LOAD DATA INFILE you can enable PrivateTmp=true for a little more security. PrivateTmp=false +# Set an explicit Start and Stop timeout of 900 seconds (15 minutes!) +# this is the same value as used in SysV init scripts in the past +# if you need a longer timeout, check the KB: +# https://mariadb.com/kb/en/library/systemd/#configuring-the-systemd-service-timeout +TimeoutStartSec=900 +TimeoutStopSec=900 + ## ## Options previously available to be set via [mysqld_safe] ## that now needs to be set by systemd config files as mysqld_safe From 3efbb5a16961a53299145075ad8312a514302afa Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 4 Dec 2019 23:40:33 +0200 Subject: [PATCH 04/38] List of unstable tests - intermediate update --- mysql-test/unstable-tests | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index c7b40236127..6ed67cdba94 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,7 +23,7 @@ # ############################################################################## # -# Based on 10.1 1bb857089fdcd3a08cb166cb6d75f3e1dbb76f27 +# Based on 10.1 05e72a33331bd9ad96ebbe4c844b94f0a5fb7390 main.alter_table_trans : MDEV-12084 - timeout main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result @@ -83,6 +83,7 @@ main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqlslap : MDEV-11801 - timeout main.mysqltest : MDEV-9269 - fails on Alpha main.old-mode : MDEV-19373 - Wrong result +main.order_by : Modified in 10.1.44 main.order_by_innodb : Modified in 10.1.42 main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock; modified in 10.1.42 @@ -97,7 +98,7 @@ main.query_cache : MDEV-12895 - Wrong result main.query_cache_debug : MDEV-15281 - Resize or similar command in progress main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away main.repair_symlink-5543 : Modified in 10.1.41 -main.selectivity : Modified in 10.1.42 +main.selectivity : Modified in 10.1.44 main.set_statement : MDEV-13183 - Wrong result main.show_explain : MDEV-10674 - sporadic failure main.sp : Modified in 10.1.42 @@ -151,6 +152,7 @@ binlog.load_data_stm_view : MDEV-16948 - Wrong result binlog_encryption.binlog_xa_recover : MDEV-12908 - Extra checkpoint binlog_encryption.encrypted_master : MDEV-12906 - Failed to sync +binlog_encryption.rpl_corruption : Include file modified in 10.1.44 binlog_encryption.rpl_parallel : MDEV-10653 - Timeout binlog_encryption.rpl_parallel_ignored_errors : Added in 10.1.42 binlog_encryption.rpl_relayrotate : MDEV-15194 - Timeout @@ -276,7 +278,7 @@ innodb.xa_recovery : MDEV-15279 - mysqld got exception innodb_fts.concurrent_insert : Modified in 10.1.42 innodb_fts.crash_recovery : Modified in 10.1.42 innodb_fts.innodb_ft_aux_table : Added in 10.1.41 -innodb_fts.innodb_fts_misc : Modified in 10.1.42 +innodb_fts.innodb_fts_misc : Modified in 10.1.44 innodb_fts.innodb_fts_misc_debug : MDEV-14156 - Unexpected warning #----------------------------------------------------------------------- @@ -337,6 +339,7 @@ perfschema.connect_attrs : MDEV-17283 - Wrong result perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown +perfschema.misc : Modified in 10.1.44 perfschema.privilege_table_io : MDEV-13184 - Extra lines perfschema.rpl_gtid_func : MDEV-16897 - Wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match @@ -376,6 +379,7 @@ rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registerin rpl.rpl_blackhole : Modified in 10.1.41 rpl.rpl_blackhole_row_annotate : Added in 10.1.41 rpl.rpl_colSize : MDEV-16112 - Server crash +rpl.rpl_corruption : Include file modified in 10.1.44 rpl.rpl_create_or_replace_fail : Added in 10.1.42 rpl.rpl_ddl : MDEV-10417 - Fails on Mips rpl.rpl_domain_id_filter_io_crash : MDEV-14357 - Wrong result @@ -429,6 +433,7 @@ rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_after_sync : MDEV-14366 - Wrong result rpl.rpl_semi_sync_after_sync_row : MDEV-14366 - Wrong result rpl.rpl_semi_sync_event_after_sync : MDEV-11806 - warnings +rpl.rpl_semi_sync_gtid_reconnect : Added in 10.1.44 rpl.rpl_semi_sync_skip_repl : Modified in 10.1.41 rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status; modified in 10.1.41 rpl.rpl_semi_sync_wait_point : MDEV-11807 - timeout in wait condition From d78f02d73d5b2f962c0ea6a1198e932c7355adc2 Mon Sep 17 00:00:00 2001 From: Axel Schwenke Date: Wed, 4 Dec 2019 14:28:13 +0100 Subject: [PATCH 05/38] MDEV-17571 Make systemd timeout behavior more compatible with long Galera SSTs Set an explicit start and stop timeout of 900 seconds for the MariaDB Server systemd service --- support-files/mariadb.service.in | 7 +++++++ support-files/mariadb@.service.in | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index ef9fa5c2a22..ede97c04112 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -120,6 +120,13 @@ UMask=007 # LOAD DATA INFILE you can enable PrivateTmp=true for a little more security. PrivateTmp=false +# Set an explicit Start and Stop timeout of 900 seconds (15 minutes!) +# this is the same value as used in SysV init scripts in the past +# Galera might need a longer timeout, check the KB if you want to change this: +# https://mariadb.com/kb/en/library/systemd/#configuring-the-systemd-service-timeout +TimeoutStartSec=900 +TimeoutStopSec=900 + ## ## Options previously available to be set via [mysqld_safe] ## that now needs to be set by systemd config files as mysqld_safe diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 8afae3bd09b..1f84b582e4e 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -141,6 +141,13 @@ UMask=007 # LOAD DATA INFILE you can enable PrivateTmp=true for a little more security. PrivateTmp=false +# Set an explicit Start and Stop timeout of 900 seconds (15 minutes!) +# this is the same value as used in SysV init scripts in the past +# if you need a longer timeout, check the KB: +# https://mariadb.com/kb/en/library/systemd/#configuring-the-systemd-service-timeout +TimeoutStartSec=900 +TimeoutStopSec=900 + ## ## Options previously available to be set via [mysqld_safe] ## that now needs to be set by systemd config files as mysqld_safe From 2b7e461cc06ea5ca167e16ff643f0b597aba118d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 5 Dec 2019 12:39:04 +0200 Subject: [PATCH 06/38] MDEV-21209 : mysql_tzinfo_to_sql's Galera checks do not work wsrep_on parameter can be visible even when wsrep_on is set OFF so we need to check variable_value from I_S also. --- .../r/mysql_tzinfo_to_sql_symlink.result | 20 +++++++++---------- .../r/mysql_tzinfo_to_sql_symlink.result | 16 +++++++-------- .../r/mysql_tzinfo_to_sql_symlink_skip.result | 8 ++++---- .../wsrep/t/mysql_tzinfo_to_sql_symlink.opt | 3 +++ .../wsrep/t/mysql_tzinfo_to_sql_symlink.test | 1 + .../t/mysql_tzinfo_to_sql_symlink_skip.opt | 3 +++ .../t/mysql_tzinfo_to_sql_symlink_skip.test | 1 + sql/tztime.cc | 10 +++++----- 8 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.opt create mode 100644 mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.opt diff --git a/mysql-test/r/mysql_tzinfo_to_sql_symlink.result b/mysql-test/r/mysql_tzinfo_to_sql_symlink.result index e33a519e897..fc9ddce08b1 100644 --- a/mysql-test/r/mysql_tzinfo_to_sql_symlink.result +++ b/mysql-test/r/mysql_tzinfo_to_sql_symlink.result @@ -4,7 +4,7 @@ # Verbose run \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=InnoDB; ALTER TABLE time_zone_name ENGINE=InnoDB; ALTER TABLE time_zone_transition ENGINE=InnoDB; @@ -36,7 +36,7 @@ ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -46,7 +46,7 @@ END IF| # Silent run \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=InnoDB; ALTER TABLE time_zone_name ENGINE=InnoDB; ALTER TABLE time_zone_transition ENGINE=InnoDB; @@ -75,7 +75,7 @@ ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -93,7 +93,7 @@ INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, ; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -105,21 +105,21 @@ END IF| # \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone_leap_second ENGINE=InnoDB; END IF| \d ; TRUNCATE TABLE time_zone_leap_second; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone_leap_second ENGINE=MyISAM; END IF| \d ; ALTER TABLE time_zone_leap_second ORDER BY Transition_time; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -131,7 +131,7 @@ END IF| # \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=InnoDB; ALTER TABLE time_zone_name ENGINE=InnoDB; ALTER TABLE time_zone_transition ENGINE=InnoDB; @@ -146,7 +146,7 @@ ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; diff --git a/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result index 9a0abd4460a..1e6ebbbd34d 100644 --- a/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result +++ b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result @@ -4,7 +4,7 @@ # Verbose run \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=InnoDB; ALTER TABLE time_zone_name ENGINE=InnoDB; ALTER TABLE time_zone_transition ENGINE=InnoDB; @@ -36,7 +36,7 @@ ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -46,7 +46,7 @@ END IF| # Silent run \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=InnoDB; ALTER TABLE time_zone_name ENGINE=InnoDB; ALTER TABLE time_zone_transition ENGINE=InnoDB; @@ -75,7 +75,7 @@ ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -93,7 +93,7 @@ INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, ; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; @@ -105,21 +105,21 @@ END IF| # \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone_leap_second ENGINE=InnoDB; END IF| \d ; TRUNCATE TABLE time_zone_leap_second; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone_leap_second ENGINE=MyISAM; END IF| \d ; ALTER TABLE time_zone_leap_second ORDER BY Transition_time; \d | IF (select count(*) from information_schema.global_variables where -variable_name='wsrep_on') = 1 THEN +variable_name='wsrep_on' and variable_value='ON') = 1 THEN ALTER TABLE time_zone ENGINE=MyISAM; ALTER TABLE time_zone_name ENGINE=MyISAM; ALTER TABLE time_zone_transition ENGINE=MyISAM; diff --git a/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink_skip.result b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink_skip.result index 4ce57c641b3..85c4d858be2 100644 --- a/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink_skip.result +++ b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink_skip.result @@ -2,7 +2,7 @@ # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above # # Verbose run -set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); +set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on' and variable_value='ON'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); prepare set_wsrep_write_binlog from @prep1; set @toggle=0; execute set_wsrep_write_binlog using @toggle; TRUNCATE TABLE time_zone; @@ -29,7 +29,7 @@ Warning: Skipping directory 'MYSQLTEST_VARDIR/zoneinfo/posix/posix': to avoid in ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time; ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; # Silent run -set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); +set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on' and variable_value='ON'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); prepare set_wsrep_write_binlog from @prep1; set @toggle=0; execute set_wsrep_write_binlog using @toggle; TRUNCATE TABLE time_zone; @@ -55,7 +55,7 @@ ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id; # # Testing with explicit timezonefile # -set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); +set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on' and variable_value='ON'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); prepare set_wsrep_write_binlog from @prep1; set @toggle=0; execute set_wsrep_write_binlog using @toggle; INSERT INTO time_zone (Use_leap_seconds) VALUES ('N'); @@ -67,7 +67,7 @@ INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, # # Testing --leap # -set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); +set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on' and variable_value='ON'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?'); prepare set_wsrep_write_binlog from @prep1; set @toggle=0; execute set_wsrep_write_binlog using @toggle; TRUNCATE TABLE time_zone_leap_second; diff --git a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.opt b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.opt new file mode 100644 index 00000000000..864f7342cc7 --- /dev/null +++ b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.opt @@ -0,0 +1,3 @@ +--wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=1 --binlog_format=ROW + + diff --git a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test index 100e09d3afb..87554635666 100644 --- a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test +++ b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test @@ -1,6 +1,7 @@ --source include/have_wsrep.inc --source include/have_symlink.inc --source include/not_windows.inc +--source include/have_innodb.inc --echo # --echo # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above diff --git a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.opt b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.opt new file mode 100644 index 00000000000..864f7342cc7 --- /dev/null +++ b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.opt @@ -0,0 +1,3 @@ +--wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=1 --binlog_format=ROW + + diff --git a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.test b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.test index bb3009bd432..ab1f94cc1cf 100644 --- a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.test +++ b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink_skip.test @@ -1,6 +1,7 @@ --source include/have_wsrep.inc --source include/have_symlink.inc --source include/not_windows.inc +--source include/have_innodb.inc --echo # --echo # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above diff --git a/sql/tztime.cc b/sql/tztime.cc index 8f66dfa0c9e..960dde38237 100644 --- a/sql/tztime.cc +++ b/sql/tztime.cc @@ -2443,7 +2443,7 @@ print_tz_leaps_as_sql(const TIME_ZONE_INFO *sp) if (!opt_skip_write_binlog) printf("\\d |\n" "IF (select count(*) from information_schema.global_variables where\n" - "variable_name='wsrep_on') = 1 THEN\n" + "variable_name='wsrep_on' and variable_value='ON') = 1 THEN\n" "ALTER TABLE time_zone_leap_second ENGINE=InnoDB;\n" "END IF|\n" "\\d ;\n"); @@ -2463,7 +2463,7 @@ print_tz_leaps_as_sql(const TIME_ZONE_INFO *sp) if (!opt_skip_write_binlog) printf("\\d |\n" "IF (select count(*) from information_schema.global_variables where\n" - "variable_name='wsrep_on') = 1 THEN\n" + "variable_name='wsrep_on' and variable_value='ON') = 1 THEN\n" "ALTER TABLE time_zone_leap_second ENGINE=MyISAM;\n" "END IF|\n" "\\d ;\n"); @@ -2719,7 +2719,7 @@ main(int argc, char **argv) sql_log_bin and wsrep_on to avoid Galera replicating below truncate table clauses. This will allow user to set different time zones to nodes in Galera cluster. */ - printf("set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?');\n" + printf("set @prep1=if((select count(*) from information_schema.global_variables where variable_name='wsrep_on' and variable_value='ON'), 'SET SESSION SQL_LOG_BIN=?, WSREP_ON=OFF;', 'do ?');\n" "prepare set_wsrep_write_binlog from @prep1;\n" "set @toggle=0; execute set_wsrep_write_binlog using @toggle;\n"); @@ -2735,7 +2735,7 @@ main(int argc, char **argv) // to allow changes to them to replicate with Galera printf("\\d |\n" "IF (select count(*) from information_schema.global_variables where\n" - "variable_name='wsrep_on') = 1 THEN\n" + "variable_name='wsrep_on' and variable_value='ON') = 1 THEN\n" "ALTER TABLE time_zone ENGINE=InnoDB;\n" "ALTER TABLE time_zone_name ENGINE=InnoDB;\n" "ALTER TABLE time_zone_transition ENGINE=InnoDB;\n" @@ -2790,7 +2790,7 @@ main(int argc, char **argv) // Fall back to MyISAM printf("\\d |\n" "IF (select count(*) from information_schema.global_variables where\n" - "variable_name='wsrep_on') = 1 THEN\n" + "variable_name='wsrep_on' and variable_value='ON') = 1 THEN\n" "ALTER TABLE time_zone ENGINE=MyISAM;\n" "ALTER TABLE time_zone_name ENGINE=MyISAM;\n" "ALTER TABLE time_zone_transition ENGINE=MyISAM;\n" From 42bad56aab25db4a05c676ba99290949ad4e1c34 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 4 Dec 2019 22:29:33 +0700 Subject: [PATCH 07/38] MDEV-21014 MTR does not detect {A,M,T,L,UB}SAN errors which happen upon server shutdown Let MTR check for error existence after running a test and return it back to user. Error reporting itset might be much better, but first of all we need to see that something went wrong. --- mysql-test/mysql-test-run.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index a44dbe92a3d..77c307727a6 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -4401,6 +4401,8 @@ sub extract_warning_lines ($$) { qr/missing DBUG_RETURN/, qr/Attempting backtrace/, qr/Assertion .* failed/, + qr/Sanitizer/, + qr/runtime error:/, ); # These are taken from the include/mtr_warnings.sql global suppression # list. They occur delayed, so they can be parsed during shutdown rather From e949b2d4abac99ffbc68bfd8c39a74e30caa3902 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Fri, 6 Dec 2019 19:56:23 +0530 Subject: [PATCH 08/38] MDEV-20959: binlog.binlog_parallel_replication_marks_row fails in buildbot with wrong result Problem: ======= Test "binlog.binlog_parallel_replication_marks_row" fails sporadically due to result length mismatch. Analysis: ========= Test generates a binary log and it looks for certain words within the binary log file and prints them. For example word like "GTID,BEGIN,COMMIT ...". Binary log output contains base64 encoded characters. Occasionally the encoded characters match with the above words and results in test failure. +XwoFWxMBAAAALgAAAGEDAAAAAB8AAAAAAAEABHRlc3QAAnQxAAIDAwACFGTIDQ== +AAAAAAAAAAAEEwQADQgICAoKCgGTIDw9 Fix: === Improve the regular expression to match exact words. --- mysql-test/include/binlog_parallel_replication_marks.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/include/binlog_parallel_replication_marks.test b/mysql-test/include/binlog_parallel_replication_marks.test index 4e673bd30c3..6e22c7dd75e 100644 --- a/mysql-test/include/binlog_parallel_replication_marks.test +++ b/mysql-test/include/binlog_parallel_replication_marks.test @@ -79,7 +79,7 @@ while () { s/end_log_pos \d+/end_log_pos #/; s/table id \d+/table id #/; s/mapped to number \d+/mapped to number #/; - print if /GTID|BEGIN|COMMIT|Table_map|Write_rows|Update_rows|Delete_rows|generated by server|40005 TEMPORARY/; + print if /\b(GTID|BEGIN|COMMIT|Table_map|Write_rows|Update_rows|Delete_rows|generated by server|40005 TEMPORARY)\b/; } close F; EOF From 8d2f6d3ca5f9a4c07cced6204c7a36028fd9a544 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 Dec 2019 16:57:59 +0100 Subject: [PATCH 09/38] Fix overly chatty connect cmake, once again --- storage/connect/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 7fc73bce59e..4ff9afd486b 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -322,10 +322,8 @@ ENDIF(CONNECT_WITH_MONGO) OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON) IF(CONNECT_WITH_REST) - MESSAGE(STATUS "=====> REST support is ON") - FIND_PACKAGE(cpprestsdk) + FIND_PACKAGE(cpprestsdk QUIET) IF (cpprestsdk_FOUND) - MESSAGE(STATUS "=====> cpprestsdk found") IF(UNIX) # INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR}) # If needed edit next line to set the path to libcpprest.so @@ -334,8 +332,6 @@ IF(CONNECT_WITH_REST) ENDIF(UNIX) SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h) add_definitions(-DREST_SUPPORT) - ELSE(NOT cpprestsdk_FOUND) - MESSAGE(STATUS "=====> cpprestsdk package not found") ENDIF (cpprestsdk_FOUND) ENDIF(CONNECT_WITH_REST) From a3a8360d577b44131577c5d40336c632d35487ae Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 12 Dec 2019 00:05:55 +0100 Subject: [PATCH 10/38] CMake,Windows - cleanup data directory prior to bootstrap for nitial_database target --- sql/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 6362ed2c561..3f6b3e694d1 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -401,9 +401,11 @@ IF(WIN32 AND MYSQLD_EXECUTABLE) MAKE_DIRECTORY(${CMAKE_CURRENT_BINARY_DIR}/data) ADD_CUSTOM_COMMAND( OUTPUT initdb.dep - COMMAND ${CMAKE_COMMAND} + COMMAND ${CMAKE_COMMAND} -E remove_directory data + COMMAND ${CMAKE_COMMAND} -E make_directory data + COMMAND ${CMAKE_COMMAND} -E chdir data ${CMAKE_COMMAND} ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DEPENDS mysqld ) ADD_CUSTOM_TARGET(initial_database From 91c3d99804e4154b87440712f8c6a5fee04c3dd0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 13 Dec 2019 11:23:04 +0100 Subject: [PATCH 11/38] tokudb: fix to compile with gcc 9.2.0 --- storage/tokudb/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 9bff7d4729f..ed802307519 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -30,6 +30,12 @@ IF (HAVE_WVLA) SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-vla") ENDIF() +CHECK_C_COMPILER_FLAG("-Wno-address-of-packed-member" HAVE_NO_ADDRESS_PACKED) +IF(HAVE_NO_ADDRESS_PACKED) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-address-of-packed-member") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-address-of-packed-member") +ENDIF() + ############################################ SET(TOKUDB_VERSION "tokudb-7.5.7") SET(TOKUDB_DEB_FILES "usr/lib/mysql/plugin/ha_tokudb.so\netc/mysql/conf.d/tokudb.cnf\nusr/bin/tokuftdump\nusr/share/doc/mariadb-server-5.5/README-TOKUDB\nusr/share/doc/mariadb-server-5.5/README.md" PARENT_SCOPE) From 794911a27a14ea83eda054cf34d94b43191aa19e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 13 Dec 2019 11:23:29 +0100 Subject: [PATCH 12/38] tokudb: disable check_huge_pages_in_practice() crashes on Debian 10 --- storage/tokudb/ft-index/portability/huge_page_detection.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/tokudb/ft-index/portability/huge_page_detection.cc b/storage/tokudb/ft-index/portability/huge_page_detection.cc index ca428987486..ea0eb568e4c 100644 --- a/storage/tokudb/ft-index/portability/huge_page_detection.cc +++ b/storage/tokudb/ft-index/portability/huge_page_detection.cc @@ -120,6 +120,7 @@ static bool check_huge_pages_config_file(const char *fname) static bool check_huge_pages_in_practice(void) // Effect: Return true if huge pages appear to be defined in practice. { + return false; // disabled, doesn't seem to work on newest distros #ifdef HAVE_MINCORE #ifdef HAVE_MAP_ANONYMOUS const int map_anonymous = MAP_ANONYMOUS; From 8129ff14407826d54745346c552fadf3d292a0d8 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 24 Jan 2019 03:06:56 -0800 Subject: [PATCH 13/38] PR #1127 and PR #1150 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR#1127: Fix is_check_constraints.result to be compatibile with 10.3 The patch is done according to the original patch for MDEV-14474 1edd09c325525cba33152 and not one which is merged on server d526679efd108478cc2af07578. This patch includes: - Rename from `is_check_constraint` to `is_check_constraints` to tests and results - Per review, change the order of fields in IS check_constraints table by adding the column `table_name` before `constraint_name`. According to the standard 2006 there is no `table_name` column. - Original patch and one in `10.3` supports embedded server this patch doesn't support. After the merge `10.3` will not support also. - Don't use patch c8b8b01b61 to change the length of `CHECK_CLAUSE` field PR#1150: MDEV-18440: Information_schema.check_constraints possible data leak This patch is extension of PR 1127 and includes: - Check for table grants - Additional test according to the MDEV specification Signed-off-by: Vicențiu Ciorbaru --- ...int.result => is_check_constraints.result} | 26 +++++++++++++ ...straint.test => is_check_constraints.test} | 33 +++++++++++++++-- sql/sql_show.cc | 37 ++++++++++++++----- 3 files changed, 83 insertions(+), 13 deletions(-) rename mysql-test/suite/funcs_1/r/{is_check_constraint.result => is_check_constraints.result} (84%) rename mysql-test/suite/funcs_1/t/{is_check_constraint.test => is_check_constraints.test} (78%) diff --git a/mysql-test/suite/funcs_1/r/is_check_constraint.result b/mysql-test/suite/funcs_1/r/is_check_constraints.result similarity index 84% rename from mysql-test/suite/funcs_1/r/is_check_constraint.result rename to mysql-test/suite/funcs_1/r/is_check_constraints.result index e36db395eb9..4d7c7b446e6 100644 --- a/mysql-test/suite/funcs_1/r/is_check_constraint.result +++ b/mysql-test/suite/funcs_1/r/is_check_constraints.result @@ -119,3 +119,29 @@ disconnect con1; connection default; DROP USER boo1; DROP USER boo2; +# +# MDEV-18440: Information_schema.check_constraints possible data leak +# +CREATE USER foo; +CREATE DATABASE db; +USE db; +CREATE TABLE t1 (a int, b int, CONSTRAINT CHECK (b > 0)); +INSERT INTO t1 VALUES (1, 2), (2, 3); +GRANT SELECT (a) ON t1 TO foo; +SHOW GRANTS FOR foo; +Grants for foo@% +GRANT USAGE ON *.* TO 'foo'@'%' +GRANT SELECT (a) ON `db`.`t1` TO 'foo'@'%' +SELECT * FROM information_schema.check_constraints; +CONSTRAINT_CATALOG CONSTRAINT_SCHEMA TABLE_NAME CONSTRAINT_NAME CHECK_CLAUSE +def db t1 CONSTRAINT_1 `b` > 0 +CONNECT con1,localhost, foo,, db; +SELECT a FROM t1; +a +1 +2 +SELECT * FROM information_schema.check_constraints; +CONSTRAINT_CATALOG CONSTRAINT_SCHEMA TABLE_NAME CONSTRAINT_NAME CHECK_CLAUSE +connection default; +DROP USER foo; +DROP DATABASE db; diff --git a/mysql-test/suite/funcs_1/t/is_check_constraint.test b/mysql-test/suite/funcs_1/t/is_check_constraints.test similarity index 78% rename from mysql-test/suite/funcs_1/t/is_check_constraint.test rename to mysql-test/suite/funcs_1/t/is_check_constraints.test index 30a72d02b34..eadfd817832 100644 --- a/mysql-test/suite/funcs_1/t/is_check_constraint.test +++ b/mysql-test/suite/funcs_1/t/is_check_constraints.test @@ -40,7 +40,7 @@ CREATE TABLE t1 CONSTRAINT CHECK (tt > 32), CONSTRAINT CHECK (tt <50),# autogenerated names table constraints CONSTRAINT CHK_tt CHECK(tt<100) # named table constraint ) ENGINE=InnoDB; - --sorted_result +--sorted_result SELECT * from information_schema.check_constraints; ALTER TABLE t1 @@ -55,7 +55,7 @@ start_date DATE, end_date DATE, CONSTRAINT CHK_dates CHECK(start_date IS NULL) #table constraint )ENGINE=Innodb; - --sorted_result +--sorted_result SELECT * from information_schema.check_constraints; ALTER TABLE t1 @@ -70,12 +70,12 @@ a int, b int check (b>0), # field constraint named 'b' CONSTRAINT b check (b>10) # table constraint ) ENGINE=InnoDB; - --sorted_result +--sorted_result SELECT * from information_schema.check_constraints; DISCONNECT con1; CONNECT(con2, localhost, boo2,, test); - --sorted_result +--sorted_result SELECT * from information_schema.check_constraints; DISCONNECT con2; @@ -90,3 +90,28 @@ DISCONNECT con1; --CONNECTION default DROP USER boo1; DROP USER boo2; + +--echo # +--echo # MDEV-18440: Information_schema.check_constraints possible data leak +--echo # + +CREATE USER foo; +CREATE DATABASE db; +USE db; +CREATE TABLE t1 (a int, b int, CONSTRAINT CHECK (b > 0)); +INSERT INTO t1 VALUES (1, 2), (2, 3); +GRANT SELECT (a) ON t1 TO foo; + +SHOW GRANTS FOR foo; +--sorted_result +SELECT * FROM information_schema.check_constraints; + +CONNECT(con1,localhost, foo,, db); +SELECT a FROM t1; +--sorted_result +SELECT * FROM information_schema.check_constraints; + +--CONNECTION default + +DROP USER foo; +DROP DATABASE db; diff --git a/sql/sql_show.cc b/sql/sql_show.cc index f54a9af5441..fcf97fc8fee 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -6526,7 +6526,7 @@ static int get_check_constraints_record(THD *thd, TABLE_LIST *tables, LEX_STRING *table_name) { DBUG_ENTER("get_check_constraints_record"); - if(res) + if (res) { if (thd->is_error()) push_warning(thd, Sql_condition::WARN_LEVEL_WARN, @@ -6535,15 +6535,32 @@ static int get_check_constraints_record(THD *thd, TABLE_LIST *tables, thd->clear_error(); DBUG_RETURN(0); } - if(!tables->view) + if (!tables->view) { StringBuffer str(system_charset_info); +#ifndef NO_EMBEDDED_ACCESS_CHECKS + TABLE_LIST table_acl_check; + bzero((char*) &table_acl_check, sizeof(table_acl_check)); +#endif for (uint i= 0; i < tables->table->s->table_check_constraints; i++) { +#ifndef NO_EMBEDDED_ACCESS_CHECKS + if (!(thd->col_access & TABLE_ACLS)) + { + table_acl_check.db= db_name->str; + table_acl_check.db_length= db_name->length; + table_acl_check.table_name= table_name->str; + table_acl_check.table_name_length= table_name->length; + table_acl_check.grant.privilege= thd->col_access; + if (check_grant(thd, TABLE_ACLS, &table_acl_check, FALSE, 1, TRUE)) + continue; + } +#endif Virtual_column_info *check= tables->table->check_constraints[i]; table->field[0]->store(STRING_WITH_LEN("def"), system_charset_info); table->field[3]->store(check->name.str, check->name.length, system_charset_info); + /* Make sure the string is empty between each print. */ str.length(0); check->print(&str); table->field[4]->store(str.ptr(), str.length(), system_charset_info); @@ -6551,8 +6568,7 @@ static int get_check_constraints_record(THD *thd, TABLE_LIST *tables, DBUG_RETURN(1); } } - - DBUG_RETURN(0); + DBUG_RETURN(res); } static int get_schema_constraints_record(THD *thd, TABLE_LIST *tables, @@ -9370,11 +9386,14 @@ ST_FIELD_INFO spatial_ref_sys_fields_info[]= ST_FIELD_INFO check_constraints_fields_info[]= { {"CONSTRAINT_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, - {"CONSTRAINT_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, + {"CONSTRAINT_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, {"TABLE_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, - {"CONSTRAINT_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, - {"CHECK_CLAUSE", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, - {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE } + {"CONSTRAINT_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, + {"CHECK_CLAUSE", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} }; /* @@ -9393,7 +9412,7 @@ ST_SCHEMA_TABLE schema_tables[]= {"CHARACTER_SETS", charsets_fields_info, 0, fill_schema_charsets, make_character_sets_old_format, 0, -1, -1, 0, 0}, {"CHECK_CONSTRAINTS", check_constraints_fields_info, 0, get_all_tables, 0, - get_check_constraints_record, 1, 2, 0, OPTIMIZE_I_S_TABLE|OPEN_TABLE_ONLY}, + get_check_constraints_record, 1, 2, 0, OPTIMIZE_I_S_TABLE|OPEN_TABLE_ONLY}, {"COLLATIONS", collation_fields_info, 0, fill_schema_collation, make_old_format, 0, -1, -1, 0, 0}, {"COLLATION_CHARACTER_SET_APPLICABILITY", coll_charset_app_fields_info, From fc860d3fa3bc854fbe6aab9179d7a4aaf6eb9edf Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 16 Dec 2019 12:57:08 +0400 Subject: [PATCH 14/38] MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set --- mysql-test/r/type_int.result | 13 +++++++++++++ mysql-test/t/type_int.test | 12 ++++++++++++ strings/ctype-simple.c | 3 ++- strings/my_strtoll10.c | 17 ++++++++++++----- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/type_int.result b/mysql-test/r/type_int.result index aaf35690306..c8b77490102 100644 --- a/mysql-test/r/type_int.result +++ b/mysql-test/r/type_int.result @@ -20,5 +20,18 @@ COALESCE(@a) 1 DROP TABLE t1; # +# MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set +# +CREATE TABLE t1 (c0 INT UNIQUE); +INSERT INTO t1 VALUES (NULL), (NULL), (NULL), (NULL), (1), (0); +SELECT * FROM t1 WHERE c0 < '\n2'; +c0 +0 +1 +DROP TABLE t1; +SELECT CAST('\n2' AS INT); +CAST('\n2' AS INT) +2 +# # End of 5.5 tests # diff --git a/mysql-test/t/type_int.test b/mysql-test/t/type_int.test index 52be12bf494..cc8b386aebe 100644 --- a/mysql-test/t/type_int.test +++ b/mysql-test/t/type_int.test @@ -14,6 +14,18 @@ SELECT COALESCE(@a:=1) FROM t1 ORDER BY STRCMP(STDDEV_SAMP(a), 'bar'); SELECT COALESCE(@a) FROM t1 ORDER BY STRCMP(STDDEV_SAMP(a), 'bar'); DROP TABLE t1; +--echo # +--echo # MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set +--echo # + +CREATE TABLE t1 (c0 INT UNIQUE); +INSERT INTO t1 VALUES (NULL), (NULL), (NULL), (NULL), (1), (0); +SELECT * FROM t1 WHERE c0 < '\n2'; +DROP TABLE t1; + +SELECT CAST('\n2' AS INT); + + --echo # --echo # End of 5.5 tests --echo # diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c index ba446a7df54..e5a1471cf63 100644 --- a/strings/ctype-simple.c +++ b/strings/ctype-simple.c @@ -1399,7 +1399,8 @@ my_strntoull10rnd_8bit(CHARSET_INFO *cs __attribute__((unused)), int shift= 0, digits= 0, negative, addon; /* Skip leading spaces and tabs */ - for ( ; str < end && (*str == ' ' || *str == '\t') ; str++); + for ( ; str < end && my_isspace(&my_charset_latin1, *str) ; ) + str++; if (str >= end) goto ret_edom; diff --git a/strings/my_strtoll10.c b/strings/my_strtoll10.c index 89450f15c9f..01c3697dcdf 100644 --- a/strings/my_strtoll10.c +++ b/strings/my_strtoll10.c @@ -98,18 +98,25 @@ longlong my_strtoll10(const char *nptr, char **endptr, int *error) if (endptr) { end= *endptr; - while (s != end && (*s == ' ' || *s == '\t')) + /* Skip leading spaces */ + for ( ; s < end && my_isspace(&my_charset_latin1, *s) ; ) s++; + if (s == end) goto no_conv; } else { endptr= &dummy; /* Easier end test */ - while (*s == ' ' || *s == '\t') - s++; - if (!*s) - goto no_conv; + /* Skip leading spaces */ + for ( ; ; s++) + { + if (!*s) + goto no_conv; + if (!my_isspace(&my_charset_latin1, *s)) + break; + } + /* This number must be big to guard against a lot of pre-zeros */ end= s+65535; /* Can't be longer than this */ } From 164cf4f4631803ceac69fdf1173d7114940d29b2 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 17 Dec 2019 21:45:53 +0530 Subject: [PATCH 15/38] MDEV-16579: Wrong result of query using DISTINCT COUNT(*) OVER (*) The query requires 2 temporary tables for execution, the window function is always attached to the last temporary table, but in this case the result field of the window function points to the first temporary table rather than the last one. Fixed this by not changing window function items with temporary table items of the first temporary table. --- mysql-test/r/win.result | 10 ++++++++++ mysql-test/t/win.test | 10 ++++++++++ sql/sql_select.cc | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index 805fd2ed3d7..2fe511d0e8b 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -3643,5 +3643,15 @@ x foo drop table t1; # +# MDEV-16579: Wrong result of query using DISTINCT COUNT(*) OVER (*) +# +CREATE TABLE t1 (i int) ; +INSERT INTO t1 VALUES (1),(0),(1),(2),(0),(1),(2),(1),(2); +SELECT DISTINCT COUNT(*) OVER (), MOD(MIN(i),2) FROM t1 GROUP BY i ; +COUNT(*) OVER () MOD(MIN(i),2) +3 0 +3 1 +drop table t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index 0f79834567b..1e302722b99 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2351,6 +2351,16 @@ INSERT INTO t1 VALUES (1),(2),(3); SELECT (SELECT MIN('foo') OVER() FROM t1 LIMIT 1) as x; drop table t1; +--echo # +--echo # MDEV-16579: Wrong result of query using DISTINCT COUNT(*) OVER (*) +--echo # + +CREATE TABLE t1 (i int) ; +INSERT INTO t1 VALUES (1),(0),(1),(2),(0),(1),(2),(1),(2); + +SELECT DISTINCT COUNT(*) OVER (), MOD(MIN(i),2) FROM t1 GROUP BY i ; +drop table t1; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index c9cb533aa33..33964cb270f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -23624,7 +23624,8 @@ change_to_use_tmp_fields(THD *thd, Ref_ptr_array ref_pointer_array, for (uint i= 0; (item= it++); i++) { Field *field; - if (item->with_sum_func && item->type() != Item::SUM_FUNC_ITEM) + if ((item->with_sum_func && item->type() != Item::SUM_FUNC_ITEM) || + item->with_window_func) item_field= item; else if (item->type() == Item::FIELD_ITEM) item_field= item->get_tmp_table_item(thd); From 0d70f40bdb554a00c2e4618cb7aefda7d36f8679 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 17 Dec 2019 16:06:15 +0100 Subject: [PATCH 16/38] Disable -Werror for rocksdb submodule We cannot have -Werror for third-party submodules because whenever they break the build we cannot fix it as needed. --- storage/rocksdb/CMakeLists.txt | 2 +- storage/rocksdb/build_rocksdb.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index fa564a603e6..cef5a8b2517 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -253,7 +253,7 @@ CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/myrocks_hotbackup.py INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/myrocks_hotbackup COMPONENT rocksdb-engine) IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET_TARGET_PROPERTIES(rocksdb_tools sst_dump mysql_ldb PROPERTIES COMPILE_FLAGS -frtti) + SET_TARGET_PROPERTIES(rocksdb_tools sst_dump mysql_ldb PROPERTIES COMPILE_FLAGS "-frtti -Wno-error") ENDIF() IF(MSVC) # RocksDB, the storage engine, overdoes "const" by adding diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index ac1307f1037..70b7c97f69c 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -471,6 +471,6 @@ list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc) ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES}) target_link_libraries(rocksdblib ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -frtti") + set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -frtti -Wno-error") endif() From 189fa3008567811706b98db0078dc4847bfd3bc7 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 17 Dec 2019 21:57:40 +0100 Subject: [PATCH 17/38] MDEV-21343 Threadpool/Unix- wait_begin() function does not wake/create threads, when it should Fixed the condition for waking up/creating another thread. If there is some work to do (if the request queue is not empty), a thread should be woken or created. The condition was incorrect since 18c9b345b43b62b7c4dbac8ce0289c1c8103c2d1 --- sql/threadpool_generic.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/threadpool_generic.cc b/sql/threadpool_generic.cc index 434857447a4..02eb336facb 100644 --- a/sql/threadpool_generic.cc +++ b/sql/threadpool_generic.cc @@ -1339,7 +1339,7 @@ void wait_begin(thread_group_t *thread_group) DBUG_ASSERT(thread_group->connection_count > 0); if ((thread_group->active_thread_count == 0) && - (is_queue_empty(thread_group) || !thread_group->listener)) + (!is_queue_empty(thread_group) || !thread_group->listener)) { /* Group might stall while this thread waits, thus wake From e9259d50f0447374764d05df50719aca352989b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 17 Dec 2019 12:10:36 +0200 Subject: [PATCH 18/38] MDEV-21335 : Galera test failure on suite wsrep Problem was that wsrep_on was OFF. --- mysql-test/suite/wsrep/disabled.def | 3 +-- mysql-test/suite/wsrep/my.cnf | 4 +--- mysql-test/suite/wsrep/t/alter_table_innodb.cnf | 12 ++++++++++++ mysql-test/suite/wsrep/t/alter_table_innodb.opt | 1 - mysql-test/suite/wsrep/t/mdev_10186.cnf | 15 +++++++++++++++ mysql-test/suite/wsrep/t/mdev_10186.opt | 1 - mysql-test/suite/wsrep/t/pool_of_threads.opt | 2 +- 7 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/wsrep/t/alter_table_innodb.cnf delete mode 100644 mysql-test/suite/wsrep/t/alter_table_innodb.opt create mode 100644 mysql-test/suite/wsrep/t/mdev_10186.cnf delete mode 100644 mysql-test/suite/wsrep/t/mdev_10186.opt diff --git a/mysql-test/suite/wsrep/disabled.def b/mysql-test/suite/wsrep/disabled.def index 862056bc83b..f4145211680 100644 --- a/mysql-test/suite/wsrep/disabled.def +++ b/mysql-test/suite/wsrep/disabled.def @@ -10,5 +10,4 @@ # ############################################################################## -foreign_key : MENT-535 Galera test failures on wsrep suite -pool_of_threads : MENT-535 Galera test failures on wsrep suite + diff --git a/mysql-test/suite/wsrep/my.cnf b/mysql-test/suite/wsrep/my.cnf index 7e51b0750a1..f65306ffca5 100644 --- a/mysql-test/suite/wsrep/my.cnf +++ b/mysql-test/suite/wsrep/my.cnf @@ -1,10 +1,8 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf -[mysqld] -wsrep-on=1 - [mysqld.1] +wsrep-on=1 #galera_port=@OPT.port #ist_port=@OPT.port #sst_port=@OPT.port diff --git a/mysql-test/suite/wsrep/t/alter_table_innodb.cnf b/mysql-test/suite/wsrep/t/alter_table_innodb.cnf new file mode 100644 index 00000000000..d8e27463cc1 --- /dev/null +++ b/mysql-test/suite/wsrep/t/alter_table_innodb.cnf @@ -0,0 +1,12 @@ +!include include/default_mysqld.cnf + +[mysqld] +wsrep-on=0 + +[mysqld.1] +wsrep-on=0 +#galera_port=@OPT.port +#ist_port=@OPT.port +#sst_port=@OPT.port +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=10M' +wsrep_cluster_address='not empty but invalid' diff --git a/mysql-test/suite/wsrep/t/alter_table_innodb.opt b/mysql-test/suite/wsrep/t/alter_table_innodb.opt deleted file mode 100644 index 1e84570d7f6..00000000000 --- a/mysql-test/suite/wsrep/t/alter_table_innodb.opt +++ /dev/null @@ -1 +0,0 @@ ---wsrep-on=0 diff --git a/mysql-test/suite/wsrep/t/mdev_10186.cnf b/mysql-test/suite/wsrep/t/mdev_10186.cnf new file mode 100644 index 00000000000..284c887648e --- /dev/null +++ b/mysql-test/suite/wsrep/t/mdev_10186.cnf @@ -0,0 +1,15 @@ +!include include/default_mysqld.cnf + +[mysqld] +wsrep-on=0 + +[mysqld.1] +wsrep-on=0 +#galera_port=@OPT.port +#ist_port=@OPT.port +#sst_port=@OPT.port +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=10M' +wsrep_cluster_address='not empty but invalid' +innodb_autoinc_lock_mode=2 +wsrep-provider=$WSREP_PROVIDER +wsrep-cluster-address=gcomm:// diff --git a/mysql-test/suite/wsrep/t/mdev_10186.opt b/mysql-test/suite/wsrep/t/mdev_10186.opt deleted file mode 100644 index e2655959c62..00000000000 --- a/mysql-test/suite/wsrep/t/mdev_10186.opt +++ /dev/null @@ -1 +0,0 @@ ---wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep-on=0 diff --git a/mysql-test/suite/wsrep/t/pool_of_threads.opt b/mysql-test/suite/wsrep/t/pool_of_threads.opt index 6948011b21b..e75bba669e8 100644 --- a/mysql-test/suite/wsrep/t/pool_of_threads.opt +++ b/mysql-test/suite/wsrep/t/pool_of_threads.opt @@ -1 +1 @@ ---innodb_autoinc_lock_mode=2 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --thread_handling=pool-of-threads wsrep-on=1 +--innodb_autoinc_lock_mode=2 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --thread_handling=pool-of-threads --wsrep-on=1 From c3824766c5788bf3aa0d1131e149dff187d41f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 18 Dec 2019 09:52:10 +0200 Subject: [PATCH 19/38] Fortify galera_partition test. --- mysql-test/suite/galera/r/galera_partition.result | 4 ++++ mysql-test/suite/galera/t/galera_partition.test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_partition.result b/mysql-test/suite/galera/r/galera_partition.result index d845de12c45..b783dbf5a3d 100644 --- a/mysql-test/suite/galera/r/galera_partition.result +++ b/mysql-test/suite/galera/r/galera_partition.result @@ -396,12 +396,16 @@ SELECT COUNT(*) FROM t1; COUNT(*) 350 connection node_2; +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; connection node_1a; +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; connection node_3; +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; connection node_4; +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; connection node_1; SET SESSION wsrep_OSU_method='RSU'; diff --git a/mysql-test/suite/galera/t/galera_partition.test b/mysql-test/suite/galera/t/galera_partition.test index a2044936cd1..00b663a4ec5 100644 --- a/mysql-test/suite/galera/t/galera_partition.test +++ b/mysql-test/suite/galera/t/galera_partition.test @@ -407,15 +407,19 @@ insert into t1 (id, dt) values (350, '2010-12-17 00:00:00'); SELECT COUNT(*) FROM t1; --connection node_2 +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); --connection node_1a +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); --connection node_3 +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); --connection node_4 +call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); --connection node_1 From 0ac0bc8cb6ac1f9b2611da63d9e06095571422fe Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 18 Dec 2019 12:34:14 +0300 Subject: [PATCH 20/38] MDEV-21341: Fix optimizer-related UBSAN failures, part #1: Fix wrong typecast --- sql/sql_select.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index ca53852fa91..40941294013 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -14157,7 +14157,7 @@ static COND* substitute_for_best_equal_field(THD *thd, JOIN_TAB *context_tab, } } else if (cond->type() == Item::FUNC_ITEM && - ((Item_cond*) cond)->functype() == Item_func::MULT_EQUAL_FUNC) + ((Item_func*) cond)->functype() == Item_func::MULT_EQUAL_FUNC) { item_equal= (Item_equal *) cond; item_equal->sort(&compare_fields_by_table_order, table_join_idx); From 8b9db11718af0c61e8d3cdbb634cf273b8998f98 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 18 Dec 2019 12:56:54 +0300 Subject: [PATCH 21/38] MDEV-21341: Fix optimizer-related UBSAN failures, part #2 Remove Query_tables_list::lock_tables_state - it is not used and it causes errors like this: sql_lex.h:1675:7: runtime error: load of value 2779096485, which is not a valid value for type 'enum_lock_tables_state' --- sql/sql_lex.h | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 4462d541e5f..cdf80daa928 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1190,28 +1190,6 @@ public: Sroutine_hash_entry **sroutines_list_own_last; uint sroutines_list_own_elements; - /** - Locking state of tables in this particular statement. - - If we under LOCK TABLES or in prelocked mode we consider tables - for the statement to be "locked" if there was a call to lock_tables() - (which called handler::start_stmt()) for tables of this statement - and there was no matching close_thread_tables() call. - - As result this state may differ significantly from one represented - by Open_tables_state::lock/locked_tables_mode more, which are always - "on" under LOCK TABLES or in prelocked mode. - */ - enum enum_lock_tables_state { - LTS_NOT_LOCKED = 0, - LTS_LOCKED - }; - enum_lock_tables_state lock_tables_state; - bool is_query_tables_locked() - { - return (lock_tables_state == LTS_LOCKED); - } - /** Number of tables which were open by open_tables() and to be locked by lock_tables(). From 984b3c15449e0b5c7b3d66047a3c490c7be40faf Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 18 Dec 2019 13:11:07 +0300 Subject: [PATCH 22/38] MDEV-21341: Fix UBSAN failures, part #3 One may not call memcpy(dst, src=NULL, size), even if size==0. --- sql/unireg.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/unireg.cc b/sql/unireg.cc index b9abe3da1b9..b116218b60e 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -297,7 +297,8 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table, pos+= reclength; int2store(pos, create_info->connect_string.length); pos+= 2; - memcpy(pos, create_info->connect_string.str, create_info->connect_string.length); + if (create_info->connect_string.length) + memcpy(pos, create_info->connect_string.str, create_info->connect_string.length); pos+= create_info->connect_string.length; int2store(pos, str_db_type.length); pos+= 2; From 3d3d8f10120dd2311ef8195d19389c4a1e85786b Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 17 Dec 2019 21:50:58 +0800 Subject: [PATCH 23/38] MDEV-21337 fix aligned_malloc() do not fallback to malloc(), always return properly aligned buffer --- storage/innobase/CMakeLists.txt | 5 ----- storage/innobase/buf/buf0buf.cc | 17 +++++++---------- storage/xtradb/CMakeLists.txt | 5 ----- storage/xtradb/buf/buf0buf.cc | 11 ++++------- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 44db4552e74..6558c3ae0dc 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -94,11 +94,6 @@ IF(NOT MSVC) SET_SOURCE_FILES_PROPERTIES(trx/trx0rec.cc PROPERTIES COMPILE_FLAGS -O1) ENDIF() - CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) - IF(HAVE_POSIX_MEMALIGN) - ADD_DEFINITIONS(-DHAVE_POSIX_MEMALIGN) - ENDIF() - # either define HAVE_IB_GCC_ATOMIC_BUILTINS or not # workaround for old gcc on x86, gcc atomic ops only work under -march=i686 IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "i686" AND CMAKE_COMPILER_IS_GNUCC AND diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 4d5b3f29f82..b6aab663f3c 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -82,19 +82,16 @@ Created 11/5/1995 Heikki Tuuri #include "snappy-c.h" #endif -inline void* aligned_malloc(size_t size, size_t align) { - void *result; +static void *aligned_malloc(size_t size, size_t align) +{ #ifdef _MSC_VER - result = _aligned_malloc(size, align); -#elif defined (HAVE_POSIX_MEMALIGN) - if(posix_memalign(&result, align, size)) { - result = 0; - } + return _aligned_malloc(size, align); #else - /* Use unaligned malloc as fallback */ - result = malloc(size); + void *result; + if (posix_memalign(&result, align, size)) + result= NULL; #endif - return result; + return result; } inline void aligned_free(void *ptr) { diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index cc269b44201..465d93eafc5 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -93,11 +93,6 @@ MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-class-memaccess") IF(NOT MSVC) - CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) - IF(HAVE_POSIX_MEMALIGN) - ADD_DEFINITIONS(-DHAVE_POSIX_MEMALIGN) - ENDIF() - # either define HAVE_IB_GCC_ATOMIC_BUILTINS or not # workaround for old gcc on x86, gcc atomic ops only work under -march=i686 IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "i686" AND CMAKE_COMPILER_IS_GNUCC AND diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 6ddcc3521da..398e1e84994 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -92,17 +92,14 @@ buf_mark_space_corrupt( /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); -inline void* aligned_malloc(size_t size, size_t align) { +static void* aligned_malloc(size_t size, size_t align) { void *result; #ifdef _MSC_VER result = _aligned_malloc(size, align); -#elif defined (HAVE_POSIX_MEMALIGN) - if(posix_memalign(&result, align, size)) { - result = 0; - } #else - /* Use unaligned malloc as fallback */ - result = malloc(size); + if(posix_memalign(&result, align, size)) { + result = NULL; + } #endif return result; } From a5a433e256d29f00b6b51babcf0aac49e95c7e82 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 18 Dec 2019 23:35:33 +0800 Subject: [PATCH 24/38] fix windows compilation --- storage/innobase/buf/buf0buf.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b6aab663f3c..136d46b7027 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -90,8 +90,8 @@ static void *aligned_malloc(size_t size, size_t align) void *result; if (posix_memalign(&result, align, size)) result= NULL; -#endif return result; +#endif } inline void aligned_free(void *ptr) { From 1f1e3ce8a18ab548f9641ea10295372abbd147ad Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 19 Dec 2019 14:03:54 +0400 Subject: [PATCH 25/38] MDEV-21319 COUNT(*) returns 1, actual SELECT returns no result in 10.3.21, but 1 result in 10.1.41 Item_ref::val_(datetime|time)_packed() erroneously called (*ref)->val_(datetime|time)_packed(). - Fixing to call (*ref)->val_(datetime|time)_packed_result(). - Backporting Item::val_(datetime|time)_packed_result() from 10.3. - Fixing Item_field::get_date_result() to handle null_value in the same way how Item_field::get_date() does. --- mysql-test/r/type_datetime.result | 81 +++++++++++++++++++++++++++++++ mysql-test/r/type_time.result | 66 +++++++++++++++++++++++++ mysql-test/t/type_datetime.test | 43 ++++++++++++++++ mysql-test/t/type_time.test | 34 +++++++++++++ sql/item.cc | 24 +++++++-- sql/item.h | 7 +++ 6 files changed, 250 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/type_datetime.result b/mysql-test/r/type_datetime.result index 591df19a9c6..3468ff67b53 100644 --- a/mysql-test/r/type_datetime.result +++ b/mysql-test/r/type_datetime.result @@ -1198,5 +1198,86 @@ Warning 1292 Incorrect datetime value: '2' for column 'pk' at row 2 DROP VIEW v1; DROP TABLE t1; # +# MDEV-21319 COUNT(*) returns 1, actual SELECT returns no result in 10.3.21, but 1 result in 10.1.41 +# +CREATE TABLE t1 +( +id INT NOT NULL PRIMARY KEY, +id2 INT, +k TINYINT, +j INT, +t DATETIME, +KEY k1 (id2,k,j,t) +); +INSERT INTO t1 VALUES +(53,54,1,0,'2019-12-13 10:09:59'), +(54,54,1,0,'2019-12-13 16:28:41'), +(55,54,1,0,'2019-12-13 16:29:10'), +(56,54,1,0,'2019-12-13 16:29:43'), +(57,54,1,0,'2019-12-13 16:30:16'), +(58,54,1,0,'2019-12-13 16:30:49'), +(59,54,1,0,'2019-12-13 16:31:23'), +(60,54,1,0,'2019-12-13 16:31:55'), +(61,54,1,0,'2019-12-13 16:32:28'), +(62,54,1,0,'2019-12-13 16:33:01'), +(63,54,1,0,'2019-12-13 16:33:34'), +(64,54,1,0,'2019-12-13 16:34:07'), +(65,54,1,0,'2019-12-13 16:34:40'), +(66,54,1,0,'2019-12-13 16:35:13'), +(67,54,1,0,'2019-12-13 16:35:46'), +(68,54,1,0,'2019-12-13 16:36:19'); +SELECT t FROM t1 GROUP BY t HAVING t=max(t); +t +2019-12-13 10:09:59 +2019-12-13 16:28:41 +2019-12-13 16:29:10 +2019-12-13 16:29:43 +2019-12-13 16:30:16 +2019-12-13 16:30:49 +2019-12-13 16:31:23 +2019-12-13 16:31:55 +2019-12-13 16:32:28 +2019-12-13 16:33:01 +2019-12-13 16:33:34 +2019-12-13 16:34:07 +2019-12-13 16:34:40 +2019-12-13 16:35:13 +2019-12-13 16:35:46 +2019-12-13 16:36:19 +SELECT t FROM t1 WHERE id2=54 and j=0 and k=1 GROUP BY t HAVING t=max(t); +t +2019-12-13 10:09:59 +2019-12-13 16:28:41 +2019-12-13 16:29:10 +2019-12-13 16:29:43 +2019-12-13 16:30:16 +2019-12-13 16:30:49 +2019-12-13 16:31:23 +2019-12-13 16:31:55 +2019-12-13 16:32:28 +2019-12-13 16:33:01 +2019-12-13 16:33:34 +2019-12-13 16:34:07 +2019-12-13 16:34:40 +2019-12-13 16:35:13 +2019-12-13 16:35:46 +2019-12-13 16:36:19 +DROP TABLE t1; +CREATE TABLE t1 (pk INT); +CREATE VIEW v1 AS SELECT * FROM t1; +INSERT INTO t1 VALUES (1); +SELECT pkis_null() || result_field->get_date(ltime,fuzzydate)) + if ((null_value= result_field->is_null()) || + result_field->get_date(ltime, fuzzydate)) { bzero((char*) ltime,sizeof(*ltime)); - return (null_value= 1); + return true; } - return (null_value= 0); + return false; } @@ -7313,7 +7327,7 @@ bool Item_ref::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) longlong Item_ref::val_datetime_packed() { DBUG_ASSERT(fixed); - longlong tmp= (*ref)->val_datetime_packed(); + longlong tmp= (*ref)->val_datetime_packed_result(); null_value= (*ref)->null_value; return tmp; } @@ -7322,7 +7336,7 @@ longlong Item_ref::val_datetime_packed() longlong Item_ref::val_time_packed() { DBUG_ASSERT(fixed); - longlong tmp= (*ref)->val_time_packed(); + longlong tmp= (*ref)->val_time_packed_result(); null_value= (*ref)->null_value; return tmp; } diff --git a/sql/item.h b/sql/item.h index fb11064f122..bea09620ef8 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1302,6 +1302,13 @@ public: uint fuzzydate= TIME_FUZZY_DATES | TIME_INVALID_DATES | TIME_TIME_ONLY; return get_date(<ime, fuzzydate) ? 0 : pack_time(<ime); } + longlong val_datetime_packed_result(); + longlong val_time_packed_result() + { + MYSQL_TIME ltime; + uint fuzzydate= TIME_TIME_ONLY | TIME_INVALID_DATES | TIME_FUZZY_DATES; + return get_date_result(<ime, fuzzydate) ? 0 : pack_time(<ime); + } // Get a temporal value in packed DATE/DATETIME or TIME format longlong val_temporal_packed(enum_field_types f_type) { From f6b58d29163adcb3b3fe1b48c56c9b80666f0b88 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Fri, 20 Dec 2019 01:16:31 +0800 Subject: [PATCH 26/38] MDEV-21239 ASAN use-after-poison in a server shutdown in innodb.innodb_buffer_pool_resize If we manually poison memory region, ASAN requires that we manually unpoison it too. We didn't do it and munmap() didn't do it too. That was the issue. os_mem_free_large(): unpoison memory region. Just in case. https://github.com/google/sanitizers/issues/182 https://github.com/llvm/llvm-project/blob/86acaa9457d3957cbe303e1e801f1e727f66ca89/compiler-rt/include/sanitizer/asan_interface.h#L21 --- storage/innobase/os/os0proc.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/innobase/os/os0proc.cc b/storage/innobase/os/os0proc.cc index 70f389436fa..60057880c18 100644 --- a/storage/innobase/os/os0proc.cc +++ b/storage/innobase/os/os0proc.cc @@ -157,6 +157,12 @@ os_mem_free_large( { ut_a(os_total_large_mem_allocated >= size); + // We could have manually poisoned that memory for ASAN. + // And we must unpoison it by ourself as specified in documentation + // for __asan_poison_memory_region() in sanitizer/asan_interface.h + // munmap() doesn't do it for us automatically. + UNIV_MEM_ALLOC(ptr, size); + #ifdef HAVE_LINUX_LARGE_PAGES if (my_use_large_pages && opt_large_page_size && !shmdt(ptr)) { my_atomic_addlint( From 8174e68895c468a2286688962a65206e677b7c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 20 Dec 2019 17:55:13 +0200 Subject: [PATCH 27/38] MDEV-21371 Assertion failure in page_rec_get_next_low() during innodb_gis.rtree_compress A debug assertion that was added in commit ed0793e096a17955c5a03844b248bcf8303dd335 turns out to be too strict. In the test innodb_gis.rtree_compress,4k the function is sometimes being invoked by purge for a spatial index root page that is not a leaf page (PAGE_LEVEL is 1). --- storage/innobase/include/page0page.ic | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 6369932db1b..98b518187b5 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -660,6 +660,7 @@ page_rec_get_next_low( } ut_ad(page_rec_is_infimum(rec) + || (!page_is_leaf(page) && !page_has_prev(page)) || !(rec_get_info_bits(page + offs, comp) & REC_INFO_MIN_REC_FLAG)); From aade6e53d398dd287ca7e771191c9975099b4fa1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 20 Dec 2019 19:47:31 +0100 Subject: [PATCH 28/38] fix a bad merge in 10.1+ one should use MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-address-of-packed-member") and it's already done in storage/tokudb/PerconaFT/CMakeLists.txt --- storage/tokudb/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 7313a4dcce4..618d257be29 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -49,12 +49,6 @@ MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-vla" DEBUG) MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-implicit-fallthrough") MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-cpp" DEBUG) -CHECK_C_COMPILER_FLAG("-Wno-address-of-packed-member" HAVE_NO_ADDRESS_PACKED) -IF(HAVE_NO_ADDRESS_PACKED) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-address-of-packed-member") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-address-of-packed-member") -ENDIF() - ############################################ SET(TOKUDB_DEB_FILES "usr/lib/mysql/plugin/ha_tokudb.so\netc/mysql/conf.d/tokudb.cnf\nusr/bin/tokuftdump" PARENT_SCOPE) MARK_AS_ADVANCED(BUILDNAME) From aaaf6ceb8bcdfec103fd3131ff8261138d8b5e39 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sat, 21 Dec 2019 13:46:33 +0400 Subject: [PATCH 29/38] MDE-21369 rpl.rpl_timezone fails with valgrind: Use of uninitialised value. It's not safe to chenge the THD::thread_id. So send the thread_id as an argument to the mysql_audit_external_lock. --- sql/sql_audit.h | 22 ++++++++++++++++------ sql/sql_insert.cc | 41 +++++++++++------------------------------ 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/sql/sql_audit.h b/sql/sql_audit.h index ff0f049779f..03f2f3aca33 100644 --- a/sql/sql_audit.h +++ b/sql/sql_audit.h @@ -291,7 +291,9 @@ void mysql_audit_notify_connection_change_user(THD *thd) } static inline -void mysql_audit_external_lock(THD *thd, TABLE_SHARE *share, int lock) +void mysql_audit_external_lock_ex(THD *thd, my_thread_id thread_id, + const char *user, const char *host, const char *ip, query_id_t query_id, + TABLE_SHARE *share, int lock) { if (lock != F_UNLCK && mysql_audit_table_enabled()) { @@ -300,14 +302,14 @@ void mysql_audit_external_lock(THD *thd, TABLE_SHARE *share, int lock) event.event_subclass= MYSQL_AUDIT_TABLE_LOCK; event.read_only= lock == F_RDLCK; - event.thread_id= (unsigned long)thd->thread_id; - event.user= sctx->user; + event.thread_id= (unsigned long)thread_id; + event.user= user; event.priv_user= sctx->priv_user; event.priv_host= sctx->priv_host; event.external_user= sctx->external_user; event.proxy_user= sctx->proxy_user; - event.host= sctx->host; - event.ip= sctx->ip; + event.host= host; + event.ip= ip; event.database= share->db.str; event.database_length= (unsigned int)share->db.length; event.table= share->table_name.str; @@ -316,12 +318,20 @@ void mysql_audit_external_lock(THD *thd, TABLE_SHARE *share, int lock) event.new_database_length= 0; event.new_table= 0; event.new_table_length= 0; - event.query_id= thd->query_id; + event.query_id= query_id; mysql_audit_notify(thd, MYSQL_AUDIT_TABLE_CLASS, &event); } } +static inline +void mysql_audit_external_lock(THD *thd, TABLE_SHARE *share, int lock) +{ + mysql_audit_external_lock_ex(thd, thd->thread_id, thd->security_ctx->user, + thd->security_ctx->host, thd->security_ctx->ip, thd->query_id, + share, lock); +} + static inline void mysql_audit_create_table(TABLE *table) { diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index a312a8652c4..ec784bc6df4 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -2076,36 +2076,16 @@ public: passed from connection thread to the handler thread. */ MDL_request grl_protection; - my_thread_id orig_thread_id; - void set_default_user() - { - thd.security_ctx->user=(char*) delayed_user; - thd.security_ctx->host=(char*) my_localhost; - thd.security_ctx->ip= NULL; - thd.query_id= 0; - thd.thread_id= orig_thread_id; - } - - void set_user_from_row(const delayed_row *r) - { - if (r) - { - thd.security_ctx->user= r->user; - thd.security_ctx->host= r->host; - thd.security_ctx->ip= r->ip; - thd.query_id= r->query_id; - thd.thread_id= r->thread_id; - } - } - Delayed_insert(SELECT_LEX *current_select) :locks_in_memory(0), thd(next_thread_id()), table(0),tables_in_use(0), stacked_inserts(0), status(0), retry(0), handler_thread_initialized(FALSE), group_count(0) { DBUG_ENTER("Delayed_insert constructor"); - orig_thread_id= thd.thread_id; - set_default_user(); + thd.security_ctx->user=(char*) delayed_user; + thd.security_ctx->host=(char*) my_localhost; + thd.security_ctx->ip= NULL; + thd.query_id= 0; strmake_buf(thd.security_ctx->priv_user, thd.security_ctx->user); thd.current_tablenr=0; thd.set_command(COM_DELAYED_INSERT); @@ -3081,7 +3061,6 @@ pthread_handler_t handle_delayed_insert(void *arg) if (di->tables_in_use && ! thd->lock && !thd->killed) { - di->set_user_from_row(di->rows.head()); /* Request for new delayed insert. Lock the table, but avoid to be blocked by a global read lock. @@ -3103,16 +3082,18 @@ pthread_handler_t handle_delayed_insert(void *arg) { delayed_row *row; I_List_iterator it(di->rows); + my_thread_id cur_thd= di->thd.thread_id; + while ((row= it++)) { - if (di->thd.thread_id != row->thread_id) + if (cur_thd != row->thread_id) { - di->set_user_from_row(row); - mysql_audit_external_lock(&di->thd, di->table->s, F_WRLCK); + mysql_audit_external_lock_ex(&di->thd, row->thread_id, + row->user, row->host, row->ip, row->query_id, + di->table->s, F_WRLCK); + cur_thd= row->thread_id; } } - di->set_default_user(); - if (di->handle_inserts()) { /* Some fatal error */ From 496532b5c54d69e012f6fc2417e97d61465588f2 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 21 Dec 2019 23:29:36 +0800 Subject: [PATCH 30/38] MDEV-20950: Fix 32-bit Windows build --- storage/innobase/include/rem0rec.h | 2 +- storage/innobase/include/rem0rec.ic | 2 +- storage/innobase/page/page0zip.cc | 2 +- storage/innobase/rem/rem0rec.cc | 3 ++- storage/innobase/row/row0merge.cc | 6 ++++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index f38efc660c4..72ee96b2887 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -455,7 +455,7 @@ value. @return offset of the start of the field, SQL null flag and extern storage flag ORed */ UNIV_INLINE -ulint +offset_t rec_2_get_field_end_info( /*=====================*/ const rec_t* rec, /*!< in: record */ diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index cb62017268c..27df29e61d6 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -883,7 +883,7 @@ value. @return offset of the start of the field, SQL null flag and extern storage flag ORed */ UNIV_INLINE -ulint +offset_t rec_2_get_field_end_info( /*=====================*/ const rec_t* rec, /*!< in: record */ diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index b466a0464f8..1c0c8f75f78 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -3171,7 +3171,7 @@ zlib_error: offsets = static_cast( mem_heap_alloc(heap, n * sizeof(ulint))); - *offsets = n; + rec_offs_set_n_alloc(offsets, n); } /* Decompress the records in heap_no order. */ diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 1d7f9880240..3abe483fd91 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -481,7 +481,8 @@ resolved: /* Old-style record: determine extra size and end offsets */ offs = REC_N_OLD_EXTRA_BYTES; if (rec_get_1byte_offs_flag(rec)) { - offs += rec_offs_n_fields(offsets); + offs += static_cast( + rec_offs_n_fields(offsets)); *rec_offs_base(offsets) = offs; /* Determine offsets to fields */ do { diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 071a65e101b..c4689e9a377 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1061,8 +1061,10 @@ row_merge_heap_create( *offsets2 = static_cast( mem_heap_alloc(heap, i * sizeof **offsets2)); - (*offsets1)[0] = (*offsets2)[0] = i; - (*offsets1)[1] = (*offsets2)[1] = dict_index_get_n_fields(index); + rec_offs_set_n_alloc(*offsets1, i); + rec_offs_set_n_alloc(*offsets2, i); + rec_offs_set_n_fields(*offsets1, dict_index_get_n_fields(index)); + rec_offs_set_n_fields(*offsets2, dict_index_get_n_fields(index)); return(heap); } From a59a015a75146ef486137de96e7a29a7eaf96a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Dec 2019 07:47:16 +0200 Subject: [PATCH 31/38] Plug memory leaks from numa_get_mems_allowed() --- storage/innobase/buf/buf0buf.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index da39e24f543..0c0f82d48b9 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -103,6 +103,7 @@ struct set_numa_interleave_t " policy to MPOL_INTERLEAVE: " << strerror(errno); } + numa_bitmask_free(numa_mems_allowed); } } @@ -1600,6 +1601,7 @@ buf_chunk_init( " buffer pool page frames to MPOL_INTERLEAVE" " (error: " << strerror(errno) << ")."; } + numa_bitmask_free(numa_mems_allowed); } #endif /* HAVE_LIBNUMA */ From bba59abb039fee1a3ee72a25643ebb7a0b64f3c5 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 23 Dec 2019 12:27:41 +0530 Subject: [PATCH 32/38] MDEV-19176 Reduce the memory usage during recovery - Moved the recv_sys->heap memory condition inside recv_parse_log_recs(). So that, InnoDB can mark the status as STORE_NO earlier. - InnoDB uses one third of buffer pool chunk size for reading the redo log records. In that case, we can avoid the scenario where buffer ran out of memory issue during recovery. --- extra/mariabackup/xtrabackup.cc | 4 +- storage/innobase/include/buf0buf.ic | 12 ++- storage/innobase/include/log0recv.h | 15 ++- storage/innobase/log/log0recv.cc | 141 +++++++++++++++++----------- 4 files changed, 112 insertions(+), 60 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index e497ba5a972..73318d121fd 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2686,7 +2686,9 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) } } - if (more_data && recv_parse_log_recs(0, STORE_NO, false)) { + store_t store = STORE_NO; + + if (more_data && recv_parse_log_recs(0, &store, 0, false)) { msg("Error: copying the log failed"); diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 98026159b8a..b3fe52a0412 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -113,7 +113,17 @@ ulint buf_pool_get_n_pages(void) /*======================*/ { - return(buf_pool_get_curr_size() / UNIV_PAGE_SIZE); + if (!buf_pool_ptr) + return buf_pool_get_curr_size() >> srv_page_size_shift; + + ulint chunk_size= 0; + for (uint i= 0; i < srv_buf_pool_instances; i++) + { + buf_pool_t* buf_pool = buf_pool_from_array(i); + for (uint j= 0; j < buf_pool->n_chunks; j++) + chunk_size+= buf_pool->chunks[j]->size; + } + return chunk_size; } /********************************************************************//** diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 74aa4fcb517..d6da4cad9a9 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -133,14 +133,19 @@ bool recv_sys_add_to_parsing_buf(const byte* log_block, lsn_t scanned_lsn); /** Parse log records from a buffer and optionally store them to a hash table to wait merging to file pages. -@param[in] checkpoint_lsn the LSN of the latest checkpoint -@param[in] store whether to store page operations -@param[in] apply whether to apply the records +@param[in] checkpoint_lsn the LSN of the latest checkpoint +@param[in] store whether to store page operations +@param[in] available_memory memory to read the redo logs +@param[in] apply whether to apply the records @return whether MLOG_CHECKPOINT record was seen the first time, or corruption was noticed */ -bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply); +bool recv_parse_log_recs( + lsn_t checkpoint_lsn, + store_t* store, + ulint available_memory, + bool apply); -/** Moves the parsing buffer data left to the buffer start. */ +/** Moves the parsing buffer data left to the buffer start */ void recv_sys_justify_left_parsing_buf(); /** Report optimized DDL operation (without redo log), diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9a20c7f7d22..347953b5289 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -761,7 +761,6 @@ recv_sys_var_init(void) recv_previous_parsed_rec_type = MLOG_SINGLE_REC_FLAG; recv_previous_parsed_rec_offset = 0; recv_previous_parsed_rec_is_multi = 0; - recv_n_pool_free_frames = 256; recv_max_page_lsn = 0; } @@ -841,17 +840,13 @@ recv_sys_init() recv_sys->flush_end = os_event_create(0); } - ulint size = buf_pool_get_curr_size(); - /* Set appropriate value of recv_n_pool_free_frames. */ - if (size >= 10 << 20) { - /* Buffer pool of size greater than 10 MB. */ - recv_n_pool_free_frames = 512; - } + recv_n_pool_free_frames = + buf_pool_get_n_pages() / 3; recv_sys->buf = static_cast( ut_malloc_nokey(RECV_PARSING_BUF_SIZE)); - recv_sys->addr_hash = hash_create(size / 512); + recv_sys->addr_hash = hash_create(buf_pool_get_curr_size() / 512); recv_sys->progress_time = time(NULL); recv_max_page_lsn = 0; @@ -2758,14 +2753,40 @@ recv_mlog_index_load(ulint space_id, ulint page_no, lsn_t lsn) } } +/** Check whether read redo log memory exceeds the available memory +of buffer pool. Store last_stored_lsn if it is not in last phase +@param[in] store whether to store page operations +@param[in] available_mem Available memory in buffer pool to + read redo logs. */ +static bool recv_sys_heap_check(store_t* store, ulint available_mem) +{ + if (*store != STORE_NO + && mem_heap_get_size(recv_sys->heap) >= available_mem) + { + if (*store == STORE_YES) + recv_sys->last_stored_lsn= recv_sys->recovered_lsn; + + *store= STORE_NO; + DBUG_PRINT("ib_log",("Ran out of memory and last " + "stored lsn " LSN_PF " last stored offset " + ULINTPF "\n",recv_sys->recovered_lsn, + recv_sys->recovered_offset)); + return true; + } + + return false; +} + /** Parse log records from a buffer and optionally store them to a hash table to wait merging to file pages. -@param[in] checkpoint_lsn the LSN of the latest checkpoint -@param[in] store whether to store page operations -@param[in] apply whether to apply the records +@param[in] checkpoint_lsn the LSN of the latest checkpoint +@param[in] store whether to store page operations +@param[in] available_mem memory to read the redo logs +@param[in] apply whether to apply the records @return whether MLOG_CHECKPOINT record was seen the first time, or corruption was noticed */ -bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply) +bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t* store, + ulint available_mem, bool apply) { byte* ptr; byte* end_ptr; @@ -2777,6 +2798,7 @@ bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply) ulint space; ulint page_no; byte* body; + const bool last_phase = (*store == STORE_IF_EXISTS); ut_ad(log_mutex_own()); ut_ad(mutex_own(&recv_sys->mutex)); @@ -2791,6 +2813,12 @@ loop: return(false); } + /* Check for memory overflow and ignore the parsing of remaining + redo log records if InnoDB ran out of memory */ + if (recv_sys_heap_check(store, available_mem) && last_phase) { + return false; + } + switch (*ptr) { case MLOG_CHECKPOINT: #ifdef UNIV_LOG_LSN_DEBUG @@ -2893,7 +2921,7 @@ loop: break; #endif /* UNIV_LOG_LSN_DEBUG */ default: - switch (store) { + switch (*store) { case STORE_NO: break; case STORE_IF_EXISTS: @@ -3077,7 +3105,7 @@ corrupted_log: recv_parse_or_apply_log_rec_body(). */ break; default: - switch (store) { + switch (*store) { case STORE_NO: break; case STORE_IF_EXISTS: @@ -3120,7 +3148,6 @@ bool recv_sys_add_to_parsing_buf(const byte* log_block, lsn_t scanned_lsn) if (!recv_sys->parse_start_lsn) { /* Cannot start parsing yet because no start point for it found */ - return(false); } @@ -3141,7 +3168,6 @@ bool recv_sys_add_to_parsing_buf(const byte* log_block, lsn_t scanned_lsn) } if (more_len == 0) { - return(false); } @@ -3176,8 +3202,9 @@ bool recv_sys_add_to_parsing_buf(const byte* log_block, lsn_t scanned_lsn) /** Moves the parsing buffer data left to the buffer start. */ void recv_sys_justify_left_parsing_buf() { - ut_memmove(recv_sys->buf, recv_sys->buf + recv_sys->recovered_offset, - recv_sys->len - recv_sys->recovered_offset); + memmove(recv_sys->buf, + recv_sys->buf + recv_sys->recovered_offset, + recv_sys->len - recv_sys->recovered_offset); recv_sys->len -= recv_sys->recovered_offset; @@ -3187,26 +3214,30 @@ void recv_sys_justify_left_parsing_buf() /** Scan redo log from a buffer and stores new log data to the parsing buffer. Parse and hash the log records if new data found. Apply log records automatically when the hash table becomes full. +@param[in] available_mem we let the hash table of recs to + grow to this size, at the maximum +@param[in,out] store_to_hash whether the records should be + stored to the hash table; this is + reset if just debug checking is + needed, or when the available_mem + runs out +@param[in] log_block log segment +@param[in] checkpoint_lsn latest checkpoint LSN +@param[in] start_lsn buffer start LSN +@param[in] end_lsn buffer end LSN +@param[in,out] contiguous_lsn it is known that all groups contain + contiguous log data upto this lsn +@param[out] group_scanned_lsn scanning succeeded upto this lsn @return true if not able to scan any more in this log group */ -static -bool -recv_scan_log_recs( -/*===============*/ - ulint available_memory,/*!< in: we let the hash table of recs - to grow to this size, at the maximum */ - store_t* store_to_hash, /*!< in,out: whether the records should be - stored to the hash table; this is reset - if just debug checking is needed, or - when the available_memory runs out */ - const byte* log_block, /*!< in: log segment */ - lsn_t checkpoint_lsn, /*!< in: latest checkpoint LSN */ - lsn_t start_lsn, /*!< in: buffer start LSN */ - lsn_t end_lsn, /*!< in: buffer end LSN */ - lsn_t* contiguous_lsn, /*!< in/out: it is known that all log - groups contain contiguous log data up - to this lsn */ - lsn_t* group_scanned_lsn)/*!< out: scanning succeeded up to - this lsn */ +static bool recv_scan_log_recs( + ulint available_mem, + store_t* store_to_hash, + const byte* log_block, + lsn_t checkpoint_lsn, + lsn_t start_lsn, + lsn_t end_lsn, + lsn_t* contiguous_lsn, + lsn_t* group_scanned_lsn) { lsn_t scanned_lsn = start_lsn; bool finished = false; @@ -3214,14 +3245,13 @@ recv_scan_log_recs( bool more_data = false; bool apply = recv_sys->mlog_checkpoint_lsn != 0; ulint recv_parsing_buf_size = RECV_PARSING_BUF_SIZE; - + const bool last_phase = (*store_to_hash == STORE_IF_EXISTS); ut_ad(start_lsn % OS_FILE_LOG_BLOCK_SIZE == 0); ut_ad(end_lsn % OS_FILE_LOG_BLOCK_SIZE == 0); ut_ad(end_lsn >= start_lsn + OS_FILE_LOG_BLOCK_SIZE); const byte* const log_end = log_block + ulint(end_lsn - start_lsn); - do { ut_ad(!finished); @@ -3332,6 +3362,13 @@ recv_scan_log_recs( = log_block_get_checkpoint_no(log_block); } + /* During last phase of scanning, there can be redo logs + left in recv_sys->buf to parse & store it in recv_sys->heap */ + if (last_phase + && recv_sys->recovered_lsn < recv_sys->scanned_lsn) { + more_data = true; + } + if (data_len < OS_FILE_LOG_BLOCK_SIZE) { /* Log data for this group ends here */ finished = true; @@ -3349,7 +3386,8 @@ recv_scan_log_recs( /* Try to parse more log records */ if (recv_parse_log_recs(checkpoint_lsn, - *store_to_hash, apply)) { + store_to_hash, available_mem, + apply)) { ut_ad(recv_sys->found_corrupt_log || recv_sys->found_corrupt_fs || recv_sys->mlog_checkpoint_lsn @@ -3358,22 +3396,18 @@ recv_scan_log_recs( goto func_exit; } - if (*store_to_hash != STORE_NO - && mem_heap_get_size(recv_sys->heap) > available_memory) { - - DBUG_PRINT("ib_log", ("Ran out of memory and last " - "stored lsn " LSN_PF, - recv_sys->recovered_lsn)); - - recv_sys->last_stored_lsn = recv_sys->recovered_lsn; - *store_to_hash = STORE_NO; - } + recv_sys_heap_check(store_to_hash, available_mem); if (recv_sys->recovered_offset > recv_parsing_buf_size / 4) { /* Move parsing buffer data to the buffer start */ - recv_sys_justify_left_parsing_buf(); } + + /* Need to re-parse the redo log which're stored + in recv_sys->buf */ + if (last_phase && *store_to_hash == STORE_NO) { + finished = false; + } } func_exit: @@ -3437,6 +3471,8 @@ recv_group_scan_log_recs( redo log records before we have finished the redo log scan. */ recv_apply_hashed_log_recs(false); + /* Rescan the redo logs from last stored lsn */ + end_lsn = recv_sys->recovered_lsn; } start_lsn = ut_uint64_align_down(end_lsn, @@ -3448,8 +3484,7 @@ recv_group_scan_log_recs( } while (end_lsn != start_lsn && !recv_scan_log_recs( available_mem, &store_to_hash, log_sys->buf, - checkpoint_lsn, - start_lsn, end_lsn, + checkpoint_lsn, start_lsn, end_lsn, contiguous_lsn, &group->scanned_lsn)); if (recv_sys->found_corrupt_log || recv_sys->found_corrupt_fs) { From 90ba87cb9e3307fff4785cadf2d058e1ce4288a4 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 23 Dec 2019 15:56:57 +0530 Subject: [PATCH 33/38] MDEV-19176 Reduce the memory usage during recovery - post-push to fix the compilation issue --- storage/innobase/include/buf0buf.ic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index b3fe52a0412..e1c8986c2ed 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -121,7 +121,7 @@ buf_pool_get_n_pages(void) { buf_pool_t* buf_pool = buf_pool_from_array(i); for (uint j= 0; j < buf_pool->n_chunks; j++) - chunk_size+= buf_pool->chunks[j]->size; + chunk_size+= buf_pool->chunks[j].size; } return chunk_size; } From 17b1b8118a4cfbd9d2a711d5c8f784c26e85864b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 23 Dec 2019 12:43:15 +0200 Subject: [PATCH 34/38] MDEV-21189 : Dropping partition with 'wsrep_OSU_method=RSU' and 'SESSION sql_log_bin = 0' cases the galera node to hang Test cleanup. Best practice for using RSU, is to isolate the node up-front, so this test did not reflect real world scenario --- mysql-test/suite/galera/r/galera_partition.result | 6 +----- mysql-test/suite/galera/t/galera_partition.test | 9 +-------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_partition.result b/mysql-test/suite/galera/r/galera_partition.result index b783dbf5a3d..c21d1005293 100644 --- a/mysql-test/suite/galera/r/galera_partition.result +++ b/mysql-test/suite/galera/r/galera_partition.result @@ -3,7 +3,6 @@ call mtr.add_suppression("WSREP: RSU failed due to pending transactions, schema: call mtr.add_suppression("WSREP: ALTER TABLE isolation failure"); connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; -connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; connection node_1; CREATE TABLE t1( id bigint unsigned NOT NULL AUTO_INCREMENT, @@ -398,9 +397,6 @@ COUNT(*) connection node_2; call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; -connection node_1a; -call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); -call p1(100);; connection node_3; call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); call p1(100);; @@ -423,6 +419,6 @@ TOI connection node_2; connection node_3; connection node_4; -connection node_1a; +connection node_1; DROP TABLE t1; DROP PROCEDURE p1; diff --git a/mysql-test/suite/galera/t/galera_partition.test b/mysql-test/suite/galera/t/galera_partition.test index 00b663a4ec5..66b2a5c10f6 100644 --- a/mysql-test/suite/galera/t/galera_partition.test +++ b/mysql-test/suite/galera/t/galera_partition.test @@ -8,7 +8,6 @@ call mtr.add_suppression("WSREP: ALTER TABLE isolation failure"); --connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 ---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connection node_1 @@ -410,10 +409,6 @@ SELECT COUNT(*) FROM t1; call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); ---connection node_1a -call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); ---send call p1(100); - --connection node_3 call mtr.add_suppression("WSREP: Sending JOIN failed:.*"); --send call p1(100); @@ -449,9 +444,7 @@ reap; --error 0,ER_LOCK_DEADLOCK reap; ---connection node_1a ---error 0,ER_LOCK_DEADLOCK -reap; +--connection node_1 DROP TABLE t1; DROP PROCEDURE p1; From cbc170adf2a0a6379591965c38d75e2d5af2c506 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Fri, 20 Dec 2019 13:33:31 +0800 Subject: [PATCH 35/38] MDEV-21362 do something with -fno-builtin-memcmp for rem0cmp.cc innodb.cmake: restrict -fno-builtin-memcmp for GCC versions older that 4.6 where optimization issue was solved. --- storage/innobase/innodb.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index 01048e9c4e8..523176b4530 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -100,7 +100,8 @@ IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU") #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wconversion") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64" OR - CMAKE_SYSTEM_PROCESSOR MATCHES "i386") + CMAKE_SYSTEM_PROCESSOR MATCHES "i386" AND + CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.6) INCLUDE(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-fno-builtin-memcmp" HAVE_NO_BUILTIN_MEMCMP) IF (HAVE_NO_BUILTIN_MEMCMP) From 3dfe1ba3b85043ed3c451292c6a47950849569dd Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 24 Dec 2019 15:05:52 +0400 Subject: [PATCH 36/38] MDEV-21388 Wrong result of DAYNAME()=xxx in combination with condition_pushdown_for_derived=on --- mysql-test/r/derived_cond_pushdown.result | 30 +++++++++++++++++++++++ mysql-test/r/func_time.result | 17 ++++++++++++- mysql-test/t/derived_cond_pushdown.test | 28 +++++++++++++++++++++ mysql-test/t/func_time.test | 8 ++++++ sql/item_timefunc.cc | 22 ++++++++++------- sql/item_timefunc.h | 25 ++++++++----------- 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index ac16dac73de..1010ed6eccb 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10551,4 +10551,34 @@ EXPLAIN } DROP TABLE t1; DROP VIEW v1; +# +# MDEV-21388 Wrong result of DAYNAME()=xxx in combination with condition_pushdown_for_derived=on +# +CREATE TABLE t1 (a INT, b DATE, c INT); +INSERT INTO t1 VALUES +(1,'2001-01-21',345), +(6,'2001-01-20',315), +(6,'2001-01-20',214); +CREATE TABLE t2 (a INT, b INT); +INSERT INTO t2 VALUES (2,19), (7,20); +CREATE VIEW v1 AS SELECT a, b, max(c) AS max_c FROM t1 +GROUP BY a,b HAVING max_c < 707; +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a); +a b max_c a b dayname(v1.b) +1 2001-01-21 345 2 19 Sunday +1 2001-01-21 345 7 20 Sunday +6 2001-01-20 315 7 20 Saturday +SET optimizer_switch='condition_pushdown_for_derived=off'; +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a) AND dayname(v1.b)='Sunday'; +a b max_c a b dayname(v1.b) +1 2001-01-21 345 2 19 Sunday +1 2001-01-21 345 7 20 Sunday +SET optimizer_switch='condition_pushdown_for_derived=on'; +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a) AND dayname(v1.b)='Sunday'; +a b max_c a b dayname(v1.b) +1 2001-01-21 345 2 19 Sunday +1 2001-01-21 345 7 20 Sunday +DROP VIEW v1; +DROP TABLE t1, t2; +SET optimizer_switch=DEFAULT; # End of 10.2 tests diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index b0c69a5334f..a82cf3c19c2 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -159,7 +159,9 @@ date_format('1999-12-31','%x-%v') date_format('2000-01-01','%x-%v') 1999-52 1999-52 select dayname("1962-03-03"),dayname("1962-03-03")+0; dayname("1962-03-03") dayname("1962-03-03")+0 -Saturday 5 +Saturday 0 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'Saturday' select monthname("1972-03-04"),monthname("1972-03-04")+0; monthname("1972-03-04") monthname("1972-03-04")+0 March 0 @@ -3508,5 +3510,18 @@ v3 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VI DROP VIEW v1,v2,v3; DROP TABLE t1,t2; # +# MDEV-21388 Wrong result of DAYNAME()=xxx in combination with condition_pushdown_for_derived=on +# +SELECT DAYNAME('2019-01-05')+0; +DAYNAME('2019-01-05')+0 +0 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'Saturday' +SELECT CAST(DAYNAME('2019-01-05') AS SIGNED); +CAST(DAYNAME('2019-01-05') AS SIGNED) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'Saturday' +# # End of 10.2 tests # diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index c9c8fb21ac7..ec15791ddbd 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -2140,4 +2140,32 @@ SELECT * FROM (SELECT 1 FROM v1 UNION (SELECT 1 FROM v1 WHERE @a := uuid())) dt; DROP TABLE t1; DROP VIEW v1; +--echo # +--echo # MDEV-21388 Wrong result of DAYNAME()=xxx in combination with condition_pushdown_for_derived=on +--echo # + +CREATE TABLE t1 (a INT, b DATE, c INT); +INSERT INTO t1 VALUES + (1,'2001-01-21',345), + (6,'2001-01-20',315), + (6,'2001-01-20',214); + +CREATE TABLE t2 (a INT, b INT); +INSERT INTO t2 VALUES (2,19), (7,20); +CREATE VIEW v1 AS SELECT a, b, max(c) AS max_c FROM t1 + GROUP BY a,b HAVING max_c < 707; + +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a); + +SET optimizer_switch='condition_pushdown_for_derived=off'; +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a) AND dayname(v1.b)='Sunday'; + +SET optimizer_switch='condition_pushdown_for_derived=on'; +SELECT *, dayname(v1.b) FROM v1,t2 WHERE (v1.max_c>214) AND (t2.a>v1.a) AND dayname(v1.b)='Sunday'; + +DROP VIEW v1; +DROP TABLE t1, t2; + +SET optimizer_switch=DEFAULT; + --echo # End of 10.2 tests diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 6c5b1cab95f..22729b4300b 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1969,6 +1969,14 @@ show create view v3; DROP VIEW v1,v2,v3; DROP TABLE t1,t2; +--echo # +--echo # MDEV-21388 Wrong result of DAYNAME()=xxx in combination with condition_pushdown_for_derived=on +--echo # + +SELECT DAYNAME('2019-01-05')+0; +SELECT CAST(DAYNAME('2019-01-05') AS SIGNED); + + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 394552f219e..b7455f36d1b 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -1090,17 +1090,21 @@ longlong Item_func_yearweek::val_int() } +static uint weekday_from_item(Item *item, bool *null_value, bool week_starts_on_sunday) +{ + MYSQL_TIME ltime; + if ((*null_value= item->get_date_with_conversion(<ime, TIME_NO_ZERO_DATE | + TIME_NO_ZERO_IN_DATE))) + return 0; + return calc_weekday(calc_daynr(ltime.year, ltime.month, ltime.day), week_starts_on_sunday) + + MY_TEST(week_starts_on_sunday); +} + + longlong Item_func_weekday::val_int() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - - if (get_arg0_date(<ime, TIME_NO_ZERO_DATE | TIME_NO_ZERO_IN_DATE)) - return 0; - - return (longlong) calc_weekday(calc_daynr(ltime.year, ltime.month, - ltime.day), - odbc_type) + MY_TEST(odbc_type); + return (longlong) weekday_from_item(args[0], &null_value, odbc_type); } bool Item_func_dayname::fix_length_and_dec() @@ -1119,7 +1123,7 @@ bool Item_func_dayname::fix_length_and_dec() String* Item_func_dayname::val_str(String* str) { DBUG_ASSERT(fixed == 1); - uint weekday=(uint) val_int(); // Always Item_func_weekday() + uint weekday= weekday_from_item(args[0], &null_value, false); const char *day_name; uint err; diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h index e37438947dd..c89b6921796 100644 --- a/sql/item_timefunc.h +++ b/sql/item_timefunc.h @@ -410,26 +410,17 @@ public: }; -class Item_func_weekday :public Item_func +class Item_func_weekday :public Item_int_func { bool odbc_type; public: Item_func_weekday(THD *thd, Item *a, bool type_arg): - Item_func(thd, a), odbc_type(type_arg) { collation.set_numeric(); } + Item_int_func(thd, a), odbc_type(type_arg) { } longlong val_int(); - double val_real() { DBUG_ASSERT(fixed == 1); return (double) val_int(); } - String *val_str(String *str) - { - DBUG_ASSERT(fixed == 1); - str->set(val_int(), &my_charset_bin); - return null_value ? 0 : str; - } const char *func_name() const { return (odbc_type ? "dayofweek" : "weekday"); } - enum Item_result result_type () const { return INT_RESULT; } - enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } bool fix_length_and_dec() { decimals= 0; @@ -447,21 +438,25 @@ public: { return get_item_copy(thd, mem_root, this); } }; -class Item_func_dayname :public Item_func_weekday +class Item_func_dayname :public Item_str_func { MY_LOCALE *locale; public: - Item_func_dayname(THD *thd, Item *a): Item_func_weekday(thd, a, 0) {} + Item_func_dayname(THD *thd, Item *a): Item_str_func(thd, a) {} const char *func_name() const { return "dayname"; } String *val_str(String *str); - enum Item_result result_type () const { return STRING_RESULT; } - enum_field_types field_type() const { return MYSQL_TYPE_VARCHAR; } bool fix_length_and_dec(); bool check_partition_func_processor(void *int_arg) {return TRUE;} bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_SESSION_FUNC); } + bool check_valid_arguments_processor(void *int_arg) + { + return !has_date_args(); + } + Item *get_copy(THD *thd, MEM_ROOT *mem_root) + { return get_item_copy(thd, mem_root, this); } }; From 891609b571a6c134d308bf7fb6f9683eb716f157 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Thu, 26 Dec 2019 12:50:21 +0530 Subject: [PATCH 37/38] MDEV-21318: Wrong results with window functions and implicit grouping The issue here is for degenerate joins we should execute the window function but it is not getting executed in all the cases. To get the window function values window function needs to be executed always. This currently does not happen in few cases where the join would return 0 or 1 row like 1) IMPOSSIBLE WHERE 2) MIN/MAX optimization 3) EMPTY CONST TABLE The fix is to make sure that window functions get executed and the temporary table is setup for the execution of window functions --- mysql-test/r/win.result | 71 +++++++++++++++++++++++++++++++++++++++++ mysql-test/t/win.test | 65 +++++++++++++++++++++++++++++++++++++ sql/sql_select.cc | 31 +++++++++++++++++- sql/sql_select.h | 1 + 4 files changed, 167 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index 2fe511d0e8b..9e551dd6dd6 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -3653,5 +3653,76 @@ COUNT(*) OVER () MOD(MIN(i),2) 3 1 drop table t1; # +# MDEV-21318: Wrong results with window functions and implicit grouping +# +CREATE TABLE t1 (a INT); +# +# With empty const table +# The expected result here is 1, NULL +# +explain +SELECT row_number() over(), sum(1) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 system NULL NULL NULL NULL 0 const row not found; Using temporary +SELECT row_number() over(), sum(1) FROM t1; +row_number() over() sum(1) +1 NULL +insert into t1 values (2); +# +# Const table has 1 row, but still impossible where +# The expected result here is 1, NULL +# +EXPLAIN SELECT row_number() over(), sum(1) FROM t1 where a=1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +SELECT row_number() over(), sum(1) FROM t1 where a=1; +row_number() over() sum(1) +1 NULL +# +# Impossible HAVING +# Empty result is expected +# +EXPLAIN SELECT row_number() over(), sum(1) FROM t1 where a=1 having 1=0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible HAVING +SELECT row_number() over(), sum(1) FROM t1 where a=1 having 1=0; +row_number() over() sum(1) +# +# const table has 1 row, no impossible where +# The expected result here is 1, 2 +# +EXPLAIN SELECT row_number() over(), sum(a) FROM t1 where a=2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 system NULL NULL NULL NULL 1 Using temporary +SELECT row_number() over(), sum(a) FROM t1 where a=2; +row_number() over() sum(a) +1 2 +drop table t1; +# +# Impossible Where +# +create table t1(a int); +insert into t1 values (1); +# +# Expected result is NULL, 0, NULL +# +EXPLAIN SELECT MAX(a) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +SELECT MAX(a) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +MAX(a) OVER () COUNT(a) abs(a) +NULL 0 NULL +# +# Expected result is 1, 0, NULL +# +EXPLAIN +SELECT MAX(1) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +SELECT MAX(1) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +MAX(1) OVER () COUNT(a) abs(a) +1 0 NULL +drop table t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index 1e302722b99..d2c0c1c9712 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2361,6 +2361,71 @@ INSERT INTO t1 VALUES (1),(0),(1),(2),(0),(1),(2),(1),(2); SELECT DISTINCT COUNT(*) OVER (), MOD(MIN(i),2) FROM t1 GROUP BY i ; drop table t1; +--echo # +--echo # MDEV-21318: Wrong results with window functions and implicit grouping +--echo # + +CREATE TABLE t1 (a INT); + +--echo # +--echo # With empty const table +--echo # The expected result here is 1, NULL +--echo # + +explain +SELECT row_number() over(), sum(1) FROM t1; +SELECT row_number() over(), sum(1) FROM t1; + +insert into t1 values (2); + +--echo # +--echo # Const table has 1 row, but still impossible where +--echo # The expected result here is 1, NULL +--echo # + +EXPLAIN SELECT row_number() over(), sum(1) FROM t1 where a=1; +SELECT row_number() over(), sum(1) FROM t1 where a=1; + +--echo # +--echo # Impossible HAVING +--echo # Empty result is expected +--echo # + +EXPLAIN SELECT row_number() over(), sum(1) FROM t1 where a=1 having 1=0; +SELECT row_number() over(), sum(1) FROM t1 where a=1 having 1=0; + +--echo # +--echo # const table has 1 row, no impossible where +--echo # The expected result here is 1, 2 +--echo # + +EXPLAIN SELECT row_number() over(), sum(a) FROM t1 where a=2; +SELECT row_number() over(), sum(a) FROM t1 where a=2; +drop table t1; + +--echo # +--echo # Impossible Where +--echo # + +create table t1(a int); +insert into t1 values (1); + +--echo # +--echo # Expected result is NULL, 0, NULL +--echo # +EXPLAIN SELECT MAX(a) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +SELECT MAX(a) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; + +--echo # +--echo # Expected result is 1, 0, NULL +--echo # + +EXPLAIN +SELECT MAX(1) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; +SELECT MAX(1) OVER (), COUNT(a), abs(a) FROM t1 WHERE FALSE; + +drop table t1; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1bf8addbcf0..72c1e876359 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1447,6 +1447,7 @@ JOIN::optimize_inner() zero_result_cause= "Zero limit"; } table_count= top_join_tab_count= 0; + handle_implicit_grouping_with_window_funcs(); error= 0; goto setup_subq_exit; } @@ -1502,6 +1503,7 @@ JOIN::optimize_inner() zero_result_cause= "No matching min/max row"; table_count= top_join_tab_count= 0; error=0; + handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } if (res > 1) @@ -1517,6 +1519,7 @@ JOIN::optimize_inner() tables_list= 0; // All tables resolved select_lex->min_max_opt_list.empty(); const_tables= top_join_tab_count= table_count; + handle_implicit_grouping_with_window_funcs(); /* Extract all table-independent conditions and replace the WHERE clause with them. All other conditions were computed by opt_sum_query @@ -1615,6 +1618,7 @@ JOIN::optimize_inner() zero_result_cause= "no matching row in const table"; DBUG_PRINT("error",("Error: %s", zero_result_cause)); error= 0; + handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } if (!(thd->variables.option_bits & OPTION_BIG_SELECTS) && @@ -1639,6 +1643,7 @@ JOIN::optimize_inner() zero_result_cause= "Impossible WHERE noticed after reading const tables"; select_lex->mark_const_derived(zero_result_cause); + handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } @@ -1781,6 +1786,7 @@ JOIN::optimize_inner() zero_result_cause= "Impossible WHERE noticed after reading const tables"; select_lex->mark_const_derived(zero_result_cause); + handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } @@ -18225,7 +18231,8 @@ void set_postjoin_aggr_write_func(JOIN_TAB *tab) } } else if (join->sort_and_group && !tmp_tbl->precomputed_group_by && - !join->sort_and_group_aggr_tab && join->tables_list) + !join->sort_and_group_aggr_tab && join->tables_list && + join->top_join_tab_count) { DBUG_PRINT("info",("Using end_write_group")); aggr->set_write_func(end_write_group); @@ -26925,6 +26932,28 @@ Item *remove_pushed_top_conjuncts(THD *thd, Item *cond) return cond; } +/* + There are 5 cases in which we shortcut the join optimization process as we + conclude that the join would be a degenerate one + 1) IMPOSSIBLE WHERE + 2) MIN/MAX optimization (@see opt_sum_query) + 3) EMPTY CONST TABLE + If a window function is present in any of the above cases then to get the + result of the window function, we need to execute it. So we need to + create a temporary table for its execution. Here we need to take in mind + that aggregate functions and non-aggregate function need not be executed. + +*/ + + +void JOIN::handle_implicit_grouping_with_window_funcs() +{ + if (select_lex->have_window_funcs() && send_row_on_empty_set()) + { + const_tables= top_join_tab_count= table_count= 0; + } +} + /** @} (end of group Query_Optimizer) */ diff --git a/sql/sql_select.h b/sql/sql_select.h index fe44f448446..ffc9eba1186 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1057,6 +1057,7 @@ protected: void restore_query_plan(Join_plan_state *restore_from); /* Choose a subquery plan for a table-less subquery. */ bool choose_tableless_subquery_plan(); + void handle_implicit_grouping_with_window_funcs(); public: JOIN_TAB *join_tab, **best_ref; From 16bce0f6fe6bcad0091dc45a97a8ac7b33fe9d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 27 Dec 2019 11:23:28 +0200 Subject: [PATCH 38/38] Cleanup: Remove dict_delete_tablespace_and_datafiles() The function was only called by innobase_drop_tablespace(), which was removed in commit 494e4b99a4a6c2f933c7e663cbb6ad5b17e8f84a and added in commit 2e814d4702d71a04388386a9f591d14a35980bfe. --- storage/innobase/dict/dict0crea.cc | 43 ---------------------------- storage/innobase/include/dict0crea.h | 10 ------- 2 files changed, 53 deletions(-) diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 4ef87dae22d..54d2eca245f 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -2460,49 +2460,6 @@ dict_replace_tablespace_in_dictionary( return(error); } -/** Delete records from SYS_TABLESPACES and SYS_DATAFILES associated -with a particular tablespace ID. -@param[in] space Tablespace ID -@param[in,out] trx Current transaction -@return DB_SUCCESS if OK, dberr_t if the operation failed */ - -dberr_t -dict_delete_tablespace_and_datafiles( - ulint space, - trx_t* trx) -{ - dberr_t err = DB_SUCCESS; - - ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_X)); - ut_ad(mutex_own(&dict_sys->mutex)); - ut_ad(srv_sys_tablespaces_open); - - trx->op_info = "delete tablespace and datafiles from dictionary"; - - pars_info_t* info = pars_info_create(); - ut_a(!is_system_tablespace(space)); - pars_info_add_int4_literal(info, "space", space); - - err = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - "DELETE FROM SYS_TABLESPACES\n" - "WHERE SPACE = :space;\n" - "DELETE FROM SYS_DATAFILES\n" - "WHERE SPACE = :space;\n" - "END;\n", - FALSE, trx); - - if (err != DB_SUCCESS) { - ib::warn() << "Could not delete space_id " - << space << " from data dictionary"; - } - - trx->op_info = ""; - - return(err); -} - /** Assign a new table ID and put it into the table cache and the transaction. @param[in,out] table Table that needs an ID @param[in,out] trx Transaction */ diff --git a/storage/innobase/include/dict0crea.h b/storage/innobase/include/dict0crea.h index 7d87686e010..359d9f556e5 100644 --- a/storage/innobase/include/dict0crea.h +++ b/storage/innobase/include/dict0crea.h @@ -238,16 +238,6 @@ dict_replace_tablespace_in_dictionary( const char* path, trx_t* trx); -/** Delete records from SYS_TABLESPACES and SYS_DATAFILES associated -with a particular tablespace ID. -@param[in] space Tablespace ID -@param[in,out] trx Current transaction -@return DB_SUCCESS if OK, dberr_t if the operation failed */ -dberr_t -dict_delete_tablespace_and_datafiles( - ulint space, - trx_t* trx); - /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */