Merge 10.8 into 10.9

client/mysql_upgrade.c

@@ -122,7 +122,6 @@ static struct my_option my_long_options[]=
    &opt_not_used, &opt_not_used, 0 , GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"host", 'h', "Connect to host.", 0,
    0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-#define PASSWORD_OPT 12
   {"password", 'p',
    "Password to use when connecting to server. If password is not given,"
    " it's solicited on the tty.", &opt_password,&opt_password,
@@ -154,7 +153,6 @@ static struct my_option my_long_options[]=
   {"upgrade-system-tables", 's', "Only upgrade the system tables in the mysql database. Tables in other databases are not checked or touched.",
    &opt_systables_only, &opt_systables_only, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-#define USER_OPT (array_elements(my_long_options) - 6)
   {"user", 'u', "User for login.", &opt_user,
    &opt_user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"verbose", 'v', "Display more output about the process; Using it twice will print connection argument; Using it 3 times will print out all CHECK, RENAME and ALTER TABLE during the check phase.",
@@ -262,11 +260,11 @@ static void print_error(const char *error_msg, DYNAMIC_STRING *output)
 */
 
 static void add_one_option_cmd_line(DYNAMIC_STRING *ds,
-                                    const struct my_option *opt,
+                                    const char *name,
-                                    const char* arg)
+                                    const char *arg)
 {
   dynstr_append(ds, "--");
-  dynstr_append(ds, opt->name);
+  dynstr_append(ds, name);
   if (arg)
   {
     dynstr_append(ds, "=");
@@ -276,10 +274,10 @@ static void add_one_option_cmd_line(DYNAMIC_STRING *ds,
 }
 
 static void add_one_option_cnf_file(DYNAMIC_STRING *ds,
-                                    const struct my_option *opt,
+                                    const char *name,
-                                    const char* arg)
+                                    const char *arg)
 {
-  dynstr_append(ds, opt->name);
+  dynstr_append(ds, name);
   if (arg)
   {
     dynstr_append(ds, "=");
@@ -327,7 +325,7 @@ get_one_option(const struct my_option *opt, const char *argument,
       */
       char *start= (char*) argument;
       /* Add password to ds_args before overwriting the arg with x's */
-      add_one_option_cnf_file(&ds_args, opt, argument);
+      add_one_option_cnf_file(&ds_args, opt->name, argument);
       while (*argument)
         *(char*)argument++= 'x';                /* Destroy argument */
       if (*start)
@@ -387,7 +385,7 @@ get_one_option(const struct my_option *opt, const char *argument,
   case OPT_MYSQL_PROTOCOL: /* --protocol */
   case OPT_PLUGIN_DIR:                          /* --plugin-dir */
   case OPT_DEFAULT_AUTH:                        /* --default-auth */
-    add_one_option_cmd_line(&conn_args, opt, argument);
+    add_one_option_cmd_line(&conn_args, opt->name, argument);
     break;
   }
 
@@ -398,7 +396,7 @@ get_one_option(const struct my_option *opt, const char *argument,
       it can be passed on to "mysql" and "mysqlcheck"
       Save it in the ds_args string
     */
-    add_one_option_cnf_file(&ds_args, opt, argument);
+    add_one_option_cnf_file(&ds_args, opt->name, argument);
   }
   return 0;
 }
@@ -1435,12 +1433,10 @@ int main(int argc, char **argv)
   {
     opt_password= get_tty_password(NullS);
     /* add password to defaults file */
-    add_one_option_cnf_file(&ds_args, &my_long_options[PASSWORD_OPT], opt_password);
+    add_one_option_cnf_file(&ds_args, "password", opt_password);
-    DBUG_ASSERT(strcmp(my_long_options[PASSWORD_OPT].name, "password") == 0);
   }
   /* add user to defaults file */
-  add_one_option_cnf_file(&ds_args, &my_long_options[USER_OPT], opt_user);
+  add_one_option_cnf_file(&ds_args, "user", opt_user);
-  DBUG_ASSERT(strcmp(my_long_options[USER_OPT].name, "user") == 0);
 
   cnf_file_path= strmov(defaults_file, "--defaults-file=");
   {

cmake/os/Windows.cmake

@@ -177,8 +177,20 @@ IF(MSVC)
    IF((NOT "${${flag}}" MATCHES "/Zi") AND (NOT "${${flag}}" MATCHES "/Z7"))
     STRING(APPEND ${flag} " /Zi")
    ENDIF()
+   # Remove inlining flags, added by CMake, if any.
+   # Compiler default is fine.
+   STRING(REGEX REPLACE "/Ob[0-3]" "" "${flag}"  "${${flag}}" )
   ENDFOREACH()
 
+  # Allow to overwrite the inlining flag
+  SET(MSVC_INLINE "" CACHE STRING
+    "MSVC Inlining option, either empty, or one of /Ob0,/Ob1,/Ob2,/Ob3")
+  IF(MSVC_INLINE MATCHES "/Ob[0-3]")
+    ADD_COMPILE_OPTIONS(${MSVC_INLINE})
+  ELSEIF(NOT(MSVC_INLINE STREQUAL ""))
+    MESSAGE(FATAL_ERROR "Invalid option for MSVC_INLINE")
+  ENDIF()
+
   IF(WITH_ASAN OR WITH_UBSAN)
     # Workaround something Linux specific
     SET(SECURITY_HARDENED 0 CACHE INTERNAL "" FORCE)

extra/mariabackup/xtrabackup.cc

@@ -1582,9 +1582,8 @@ struct my_option xb_server_options[] =
   {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
    "Ignored for mysqld option compatibility",
    (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
-   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20,
+   GET_ULL, REQUIRED_ARG, 96 << 20, 4 << 20,
-   std::numeric_limits<ulonglong>::max(), 0,
+   std::numeric_limits<ulonglong>::max(), 0, 4096, 0},
-   UNIV_PAGE_SIZE_MAX, 0},
   {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
    "Path to InnoDB log files.", &srv_log_group_home_dir,
    &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -2540,10 +2539,10 @@ void xtrabackup_io_throttling()
   if (!xtrabackup_backup)
     return;
 
-  mysql_mutex_lock(&log_sys.mutex);
+  mysql_mutex_lock(&recv_sys.mutex);
   if (xtrabackup_throttle && (io_ticket--) < 0)
-    mysql_cond_wait(&wait_throttle, &log_sys.mutex);
+    mysql_cond_wait(&wait_throttle, &recv_sys.mutex);
-  mysql_mutex_unlock(&log_sys.mutex);
+  mysql_mutex_unlock(&recv_sys.mutex);
 }
 
 static
@@ -2972,7 +2971,7 @@ skip:
 @return	whether the operation failed */
 static bool xtrabackup_copy_logfile()
 {
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  mysql_mutex_assert_owner(&recv_sys.mutex);
   DBUG_EXECUTE_IF("log_checksum_mismatch", return false;);
 
   ut_a(dst_log_file);
@@ -2980,7 +2979,6 @@ static bool xtrabackup_copy_logfile()
   const size_t sequence_offset{log_sys.is_encrypted() ? 8U + 5U : 5U};
   const size_t block_size_1{log_sys.get_block_size() - 1};
 
-  mysql_mutex_lock(&recv_sys.mutex);
 #ifdef HAVE_PMEM
   if (log_sys.is_pmem())
   {
@@ -3003,7 +3001,7 @@ static bool xtrabackup_copy_logfile()
 #ifdef HAVE_PMEM
     if (log_sys.is_pmem())
     {
-      if ((ut_d(r=) recv_sys.parse_pmem(STORE_IF_EXISTS)) != recv_sys_t::OK)
+      if ((ut_d(r=) recv_sys.parse_pmem(STORE_NO)) != recv_sys_t::OK)
       {
         ut_ad(r == recv_sys_t::GOT_EOF);
         goto retry;
@@ -3073,7 +3071,7 @@ static bool xtrabackup_copy_logfile()
 
         start_offset= recv_sys.offset;
       }
-      while ((ut_d(r=)recv_sys.parse_pmem(STORE_IF_EXISTS)) == recv_sys_t::OK);
+      while ((ut_d(r=)recv_sys.parse_pmem(STORE_NO)) == recv_sys_t::OK);
 
       ut_ad(r == recv_sys_t::GOT_EOF);
       pthread_cond_broadcast(&scanned_lsn_cond);
@@ -3109,7 +3107,7 @@ static bool xtrabackup_copy_logfile()
       if (log_sys.buf[recv_sys.offset] <= 1)
         break;
 
-      if (recv_sys.parse_mtr(STORE_IF_EXISTS) == recv_sys_t::OK)
+      if (recv_sys.parse_mtr(STORE_NO) == recv_sys_t::OK)
       {
         do
         {
@@ -3119,7 +3117,7 @@ static bool xtrabackup_copy_logfile()
                                                  sequence_offset));
           *seq= 1;
         }
-        while ((r= recv_sys.parse_mtr(STORE_IF_EXISTS)) == recv_sys_t::OK);
+        while ((r= recv_sys.parse_mtr(STORE_NO)) == recv_sys_t::OK);
 
         if (ds_write(dst_log_file, log_sys.buf + start_offset,
                      recv_sys.offset - start_offset))
@@ -3127,7 +3125,6 @@ static bool xtrabackup_copy_logfile()
 #ifdef HAVE_PMEM
         write_error:
 #endif
-          mysql_mutex_unlock(&recv_sys.mutex);
           msg("Error: write to ib_logfile0 failed");
           return true;
         }
@@ -3148,12 +3145,11 @@ static bool xtrabackup_copy_logfile()
         if (recv_sys.offset < log_sys.get_block_size())
           break;
 
-        mysql_mutex_unlock(&recv_sys.mutex);
-
         if (xtrabackup_throttle && io_ticket-- < 0)
-          mysql_cond_wait(&wait_throttle, &log_sys.mutex);
+          mysql_cond_wait(&wait_throttle, &recv_sys.mutex);
 
         retry_count= 0;
+        continue;
       }
       else
       {
@@ -3173,7 +3169,6 @@ static bool xtrabackup_copy_logfile()
     mysql_mutex_lock(&recv_sys.mutex);
   }
 
-  mysql_mutex_unlock(&recv_sys.mutex);
   msg(">> log scanned up to (" LSN_PF ")", recv_sys.lsn);
   return false;
 }
@@ -3204,16 +3199,16 @@ extern lsn_t server_lsn_after_lock;
 static void log_copying_thread()
 {
   my_thread_init();
-  mysql_mutex_lock(&log_sys.mutex);
+  mysql_mutex_lock(&recv_sys.mutex);
   while (!xtrabackup_copy_logfile() &&
          (!metadata_to_lsn || metadata_to_lsn > recv_sys.lsn))
   {
     timespec abstime;
     set_timespec_nsec(abstime, 1000ULL * xtrabackup_log_copy_interval);
-    mysql_cond_timedwait(&log_copying_stop, &log_sys.mutex, &abstime);
+    mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime);
   }
   log_copying_running= false;
-  mysql_mutex_unlock(&log_sys.mutex);
+  mysql_mutex_unlock(&recv_sys.mutex);
   my_thread_end();
 }
 
@@ -3226,13 +3221,13 @@ static void *io_watching_thread(void*)
   /* currently, for --backup only */
   ut_a(xtrabackup_backup);
 
-  mysql_mutex_lock(&log_sys.mutex);
+  mysql_mutex_lock(&recv_sys.mutex);
 
   while (log_copying_running && !metadata_to_lsn)
   {
     timespec abstime;
     set_timespec(abstime, 1);
-    mysql_cond_timedwait(&log_copying_stop, &log_sys.mutex, &abstime);
+    mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime);
     io_ticket= xtrabackup_throttle;
     mysql_cond_broadcast(&wait_throttle);
   }
@@ -3240,7 +3235,7 @@ static void *io_watching_thread(void*)
   /* stop io throttle */
   xtrabackup_throttle= 0;
   mysql_cond_broadcast(&wait_throttle);
-  mysql_mutex_unlock(&log_sys.mutex);
+  mysql_mutex_unlock(&recv_sys.mutex);
   return nullptr;
 }
 
@@ -4512,7 +4507,7 @@ static void stop_backup_threads(bool running)
 @return	whether the operation succeeded */
 static bool xtrabackup_backup_low()
 {
-	mysql_mutex_lock(&log_sys.mutex);
+	mysql_mutex_lock(&recv_sys.mutex);
 	ut_ad(!metadata_to_lsn);
 
 	/* read the latest checkpoint lsn */
@@ -4531,19 +4526,19 @@ static bool xtrabackup_backup_low()
 		recv_sys.lsn = lsn;
 		mysql_cond_broadcast(&log_copying_stop);
 		const bool running= log_copying_running;
-		mysql_mutex_unlock(&log_sys.mutex);
+		mysql_mutex_unlock(&recv_sys.mutex);
 		stop_backup_threads(running);
-		mysql_mutex_lock(&log_sys.mutex);
+		mysql_mutex_lock(&recv_sys.mutex);
 	}
 
 	if (metadata_to_lsn && xtrabackup_copy_logfile()) {
-		mysql_mutex_unlock(&log_sys.mutex);
+		mysql_mutex_unlock(&recv_sys.mutex);
 		ds_close(dst_log_file);
 		dst_log_file = NULL;
 		return false;
 	}
 
-	mysql_mutex_unlock(&log_sys.mutex);
+	mysql_mutex_unlock(&recv_sys.mutex);
 
 	if (ds_close(dst_log_file) || !metadata_to_lsn) {
 		dst_log_file = NULL;
@@ -4632,10 +4627,10 @@ static bool xtrabackup_backup_func()
         if(innodb_init_param()) {
 fail:
 		if (log_copying_running) {
-			mysql_mutex_lock(&log_sys.mutex);
+			mysql_mutex_lock(&recv_sys.mutex);
 			metadata_to_lsn = 1;
 			mysql_cond_broadcast(&log_copying_stop);
-			mysql_mutex_unlock(&log_sys.mutex);
+			mysql_mutex_unlock(&recv_sys.mutex);
 			stop_backup_threads(true);
 		}
 
@@ -4692,12 +4687,12 @@ fail:
 	log_sys.create();
 	/* get current checkpoint_lsn */
 
-	mysql_mutex_lock(&log_sys.mutex);
+	mysql_mutex_lock(&recv_sys.mutex);
 
 	if (recv_sys.find_checkpoint() != DB_SUCCESS) {
 		msg("Error: cannot read redo log header");
 unlock_and_fail:
-		mysql_mutex_unlock(&log_sys.mutex);
+		mysql_mutex_unlock(&recv_sys.mutex);
 free_and_fail:
 		aligned_free(const_cast<byte*>(field_ref_zero));
 		field_ref_zero = nullptr;
@@ -4710,7 +4705,7 @@ free_and_fail:
 	}
 
 	recv_needed_recovery = true;
-	mysql_mutex_unlock(&log_sys.mutex);
+	mysql_mutex_unlock(&recv_sys.mutex);
 
 	/* create extra LSN dir if it does not exist. */
 	if (xtrabackup_extra_lsndir
@@ -4772,12 +4767,12 @@ free_and_fail:
 
 	/* copy log file by current position */
 
-	mysql_mutex_lock(&log_sys.mutex);
+	mysql_mutex_lock(&recv_sys.mutex);
 	recv_sys.lsn = log_sys.next_checkpoint_lsn;
 
 	const bool log_copy_failed = xtrabackup_copy_logfile();
 
-	mysql_mutex_unlock(&log_sys.mutex);
+	mysql_mutex_unlock(&recv_sys.mutex);
 
 	if (log_copy_failed) {
 		log_copying_running = false;
@@ -5212,7 +5207,7 @@ xb_delta_open_matching_space(
 		return OS_FILE_CLOSED;
 	}
 
-	mysql_mutex_lock(&log_sys.mutex);
+	mysql_mutex_lock(&recv_sys.mutex);
 	if (!fil_is_user_tablespace_id(info.space_id)) {
 found:
 		/* open the file and return its handle */
@@ -5225,7 +5220,7 @@ found:
 			msg("mariabackup: Cannot open file %s\n", real_name);
 		}
 exit:
-		mysql_mutex_unlock(&log_sys.mutex);
+		mysql_mutex_unlock(&recv_sys.mutex);
 		return file;
 	}
 

include/my_alarm.h

@@ -31,7 +31,7 @@ extern ulong my_time_to_wait_for_lock;
 #include <signal.h>
 #ifdef HAVE_SIGHANDLER_T
 #define sig_return sighandler_t
-#elif defined(SOLARIS) || defined(__sun) || defined(__APPLE__) || defined(__FreeBSD__)
+#elif defined(SOLARIS) || defined(__sun) || defined(__APPLE__) || defined(__FreeBSD__) || defined(_AIX)
 typedef void (*sig_return)(int); /* Returns type from signal */
 #else
 typedef void (*sig_return)(void); /* Returns type from signal */

include/providers/lzma.h

@@ -52,12 +52,18 @@ typedef enum
 
 #define lzma_stream_buffer_decode(...) provider_service_lzma->lzma_stream_buffer_decode_ptr (__VA_ARGS__)
 #define lzma_easy_buffer_encode(...)   provider_service_lzma->lzma_easy_buffer_encode_ptr   (__VA_ARGS__)
+#elif LZMA_VERSION < 50010030
+#define lzma_maybe_const
+#endif
+
+#ifndef lzma_maybe_const
+#define lzma_maybe_const const
 #endif
 
 #define DEFINE_lzma_stream_buffer_decode(NAME) NAME( \
     uint64_t *memlimit,                              \
     uint32_t flags,                                  \
-    const lzma_allocator *allocator,                 \
+    lzma_maybe_const lzma_allocator *allocator,      \
     const uint8_t *in,                               \
     size_t *in_pos,                                  \
     size_t in_size,                                  \
@@ -69,7 +75,7 @@ typedef enum
 #define DEFINE_lzma_easy_buffer_encode(NAME) NAME(   \
     uint32_t preset,                                 \
     lzma_check check,                                \
-    const lzma_allocator *allocator,                 \
+    lzma_maybe_const lzma_allocator *allocator,      \
     const uint8_t *in,                               \
     size_t in_size,                                  \
     uint8_t *out,                                    \

mysql-test/main/backup_locks.result

@@ -39,6 +39,28 @@ MDL_INTENTION_EXCLUSIVE	Schema metadata lock	test
 select * from t1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 backup unlock;
+connection con1;
+connection default;
+#
+# Check that BACKUP LOCK blocks some operations
+#
+create sequence seq1;
+create sequence seq2;
+backup lock seq1;
+connection con1;
+CREATE OR REPLACE SEQUENCE seq1 START -28;
+ERROR HY000: Sequence 'test.seq1' values are conflicting
+SET STATEMENT max_statement_time=10 FOR CREATE OR REPLACE SEQUENCE seq1 START 50;
+ERROR 70100: Query execution was interrupted (max_statement_time exceeded)
+SET STATEMENT max_statement_time=10 FOR ALTER SEQUENCE IF EXISTS seq1 NOMAXVALUE;
+ERROR 70100: Query execution was interrupted (max_statement_time exceeded)
+SET STATEMENT max_statement_time=10 FOR ALTER SEQUENCE IF EXISTS seq1 MAXVALUE 1000;
+ERROR 70100: Query execution was interrupted (max_statement_time exceeded)
+SET STATEMENT max_statement_time=10 for rename table seq2 to seq3, seq3 to seq1;
+ERROR 70100: Query execution was interrupted (max_statement_time exceeded)
+connection default;
+backup unlock;
+drop table seq1,seq2;
 #
 # BACKUP LOCK and BACKUP UNLOCK are not allowed in procedures.
 #
@@ -141,7 +163,6 @@ ERROR HY000: Can't execute the given command because you have active locked tabl
 SET STATEMENT max_statement_time=180 FOR BACKUP LOCK test.u;
 # restart
 #
-connection con1;
 connection default;
 disconnect con1;
 show tables;

mysql-test/main/backup_locks.test

@@ -43,10 +43,39 @@ SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.me
 --error ER_LOCK_DEADLOCK
 select * from t1;
 backup unlock;
+connection con1;
+--reap
+connection default;
+
+--echo #
+--echo # Check that BACKUP LOCK blocks some operations
+--echo #
+
+# These test has to be done with timeouts as we want to ensure that the tables
+# doesn't change
+
+create sequence seq1;
+create sequence seq2;
+backup lock seq1;
+connection con1;
+--error ER_SEQUENCE_INVALID_DATA
+CREATE OR REPLACE SEQUENCE seq1 START -28;
+--error ER_STATEMENT_TIMEOUT
+SET STATEMENT max_statement_time=10 FOR CREATE OR REPLACE SEQUENCE seq1 START 50;
+--error ER_STATEMENT_TIMEOUT
+SET STATEMENT max_statement_time=10 FOR ALTER SEQUENCE IF EXISTS seq1 NOMAXVALUE;
+--error ER_STATEMENT_TIMEOUT
+SET STATEMENT max_statement_time=10 FOR ALTER SEQUENCE IF EXISTS seq1 MAXVALUE 1000;
+--error ER_STATEMENT_TIMEOUT
+SET STATEMENT max_statement_time=10 for rename table seq2 to seq3, seq3 to seq1;
+connection default;
+backup unlock;
+drop table seq1,seq2;
 
 --echo #
 --echo # BACKUP LOCK and BACKUP UNLOCK are not allowed in procedures.
 --echo #
+
 delimiter |;
 --error ER_SP_BADSTATEMENT
 CREATE PROCEDURE p_BACKUP_LOCK()
@@ -162,8 +191,6 @@ SET STATEMENT max_statement_time=180 FOR BACKUP LOCK test.u;
 
 --echo #
 
-connection con1;
---reap
 connection default;
 disconnect con1;
 show tables;

mysql-test/main/cte_nonrecursive.result

@@ -2185,6 +2185,39 @@ select * from t1;
 a
 7
 drop table t1,t2;
+#
+# MDEV-25766: Unused CTE lead to a crash in
+# find_field_in_tables/find_order_in_list
+#
+create table t1 (f1 INTEGER);
+create view v1 as
+select
+subq_0.c4 as c2,
+subq_0.c4 as c4
+from
+(select
+ref_0.f1 as c4
+from
+t1 as ref_0
+where (select 1)
+) as subq_0
+order by c2, c4 desc;
+WITH
+unused_with AS (select
+subq_0.c4 as c6
+from
+(select
+11 as c4
+from
+v1 as ref_0
+) as subq_0,
+v1 as ref_2
+)
+select 1 ;
+1
+1
+drop view v1;
+drop table t1;
 # End of 10.2 tests
 #
 # MDEV-21673: several references to CTE that uses

mysql-test/main/cte_nonrecursive.test

@@ -1637,6 +1637,42 @@ select * from t1;
 
 drop table t1,t2;
 
+--echo #
+--echo # MDEV-25766: Unused CTE lead to a crash in
+--echo # find_field_in_tables/find_order_in_list
+--echo #
+
+create table t1 (f1 INTEGER);
+
+create view v1 as
+select
+    subq_0.c4 as c2,
+    subq_0.c4 as c4
+  from
+    (select
+          ref_0.f1 as c4
+        from
+          t1 as ref_0
+        where (select 1)
+        ) as subq_0
+  order by c2, c4 desc;
+
+WITH
+unused_with AS (select
+    subq_0.c4 as c6
+  from
+    (select
+            11 as c4
+          from
+            v1 as ref_0
+     ) as subq_0,
+     v1 as ref_2
+)
+select 1 ;
+
+drop view v1;
+drop table t1;
+
 --echo # End of 10.2 tests
 
 --echo #

mysql-test/main/group_min_max.result

@@ -4028,6 +4028,43 @@ drop table t1;
 # End of 10.1 tests
 #
 #
+# MDEV-27442 Wrong result upon query with DISTINCT and EXISTS subquery
+#
+CREATE TABLE t1 (a int, b int, KEY b (b,a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (0,100),(2,100),(2,101),(3,102);
+# Must not use Using index for group-by
+explain SELECT DISTINCT b FROM t1 WHERE EXISTS ( SELECT 1 FROM DUAL WHERE a > 1 );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	index	NULL	b	10	NULL	4	Using where; Using index
+2	DEPENDENT SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+SELECT DISTINCT b FROM t1 WHERE EXISTS ( SELECT 1 FROM DUAL WHERE a > 1 );
+b
+100
+101
+102
+DROP TABLE t1;
+#
+# MDEV-26585 Wrong query results when `using index for group-by`
+#
+CREATE TABLE `t1` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`owner_id` int(11) DEFAULT NULL,
+`foo` tinyint(1) DEFAULT 0,
+`whatever` varchar(255) DEFAULT NULL,
+PRIMARY KEY (`id`),
+KEY `index_t1_on_owner_id_and_foo` (`owner_id`,`foo`)
+) engine=InnoDB DEFAULT CHARSET=utf8;
+INSERT INTO t1 (owner_id, foo, whatever)
+VALUES (1, TRUE, "yello"), (1, FALSE, "yello"), (2, TRUE, "yello"),
+(2, TRUE, "yello"), (2, FALSE, "yello");
+EXPLAIN SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	index_t1_on_owner_id_and_foo	7	NULL	5	Using where; Using index
+SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1);
+owner_id
+1
+DROP TABLE t1;
+#
 # MDEV-24353: Adding GROUP BY slows down a query
 #
 CREATE TABLE t1 (p int NOT NULL, a int NOT NULL, PRIMARY KEY (p,a));

mysql-test/main/group_min_max.test

@@ -5,7 +5,7 @@
 
 --source include/default_optimizer_switch.inc
 --source include/have_sequence.inc
-
+--source include/have_innodb.inc
 #
 # TODO:
 # Add queries with:
@@ -1691,6 +1691,37 @@ drop table t1;
 --echo # End of 10.1 tests
 --echo #
 
+--echo #
+--echo # MDEV-27442 Wrong result upon query with DISTINCT and EXISTS subquery
+--echo #
+
+CREATE TABLE t1 (a int, b int, KEY b (b,a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (0,100),(2,100),(2,101),(3,102);
+--echo # Must not use Using index for group-by
+explain SELECT DISTINCT b FROM t1 WHERE EXISTS ( SELECT 1 FROM DUAL WHERE a > 1 );
+SELECT DISTINCT b FROM t1 WHERE EXISTS ( SELECT 1 FROM DUAL WHERE a > 1 );
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-26585 Wrong query results when `using index for group-by`
+--echo #
+
+CREATE TABLE `t1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `owner_id` int(11) DEFAULT NULL,
+  `foo` tinyint(1) DEFAULT 0,
+  `whatever` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `index_t1_on_owner_id_and_foo` (`owner_id`,`foo`)
+) engine=InnoDB DEFAULT CHARSET=utf8;
+
+INSERT INTO t1 (owner_id, foo, whatever)
+VALUES (1, TRUE, "yello"), (1, FALSE, "yello"), (2, TRUE, "yello"),
+       (2, TRUE, "yello"), (2, FALSE, "yello");
+EXPLAIN SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1);
+SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1);
+DROP TABLE t1;
+
 --echo #
 --echo # MDEV-24353: Adding GROUP BY slows down a query
 --echo #

mysql-test/main/subselect4.result

@@ -1358,6 +1358,8 @@ INSERT IGNORE INTO t2 VALUES (8,0,0),(5,0,0);
 CREATE TABLE t3 (f4 int,KEY (f4)) ;
 INSERT IGNORE INTO t3 VALUES (0),(0);
 set @@optimizer_switch='semijoin=off';
+# NOTE: the following should have 'SUBQUERY', not 'DEPENDENT SUBQUERY'
+# for line with id=2, see MDEV-27794.
 EXPLAIN
 SELECT * FROM t1 WHERE
 (SELECT f2 FROM t2
@@ -1367,7 +1369,7 @@ FROM t3 AS SQ1_t1 JOIN t3 AS SQ1_t3 ON SQ1_t3.f4
 GROUP BY SQ1_t1.f4));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	NULL	NULL	NULL	NULL	1	
-2	SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 3	SUBQUERY	SQ1_t3	range	f4	f4	5	NULL	2	Using where; Using index; Using temporary
 3	SUBQUERY	SQ1_t1	index	NULL	f4	5	NULL	2	Using index; Using join buffer (flat, BNL join)
 SELECT * FROM t1 WHERE

mysql-test/main/subselect4.test

@@ -1041,6 +1041,8 @@ INSERT IGNORE INTO t3 VALUES (0),(0);
 
 set @@optimizer_switch='semijoin=off';
 
+--echo # NOTE: the following should have 'SUBQUERY', not 'DEPENDENT SUBQUERY'
+--echo # for line with id=2, see MDEV-27794.
 EXPLAIN
 SELECT * FROM t1 WHERE
 (SELECT f2 FROM t2

mysql-test/main/union_innodb.result

@@ -0,0 +1,34 @@
+#
+# MDEV-25636: Bug report: abortion in sql/sql_parse.cc:6294
+#
+CREATE TABLE t1 (i1 int)engine=innodb;
+INSERT INTO `t1` VALUES (62),(66);
+CREATE TABLE t2 (i1 int) engine=innodb;
+SELECT 1 FROM t1 
+WHERE t1.i1 =( SELECT t1.i1 FROM t2  
+UNION SELECT i1 FROM (t1 AS dt1 natural JOIN t2) 
+window w1 as (partition by t1.i1));
+1
+drop table t1,t2;
+# Another testcase
+CREATE TABLE t1 (i3 int NOT NULL, i1 int , i2 int , i4 int , PRIMARY key(i2));
+INSERT INTO t1 VALUES (6,72,98,98),(46,1,6952,0);
+SELECT i1 FROM t1 
+WHERE t1.i3 =
+(SELECT ref_4.i2 FROM t1 AS ref_4
+WHERE t1.i2 > (SELECT i3 FROM t1 ORDER BY i3 LIMIT 1 OFFSET 4)
+UNION 
+SELECT ref_6.i2
+FROM (t1 AS ref_5 JOIN t1 AS ref_6 ON ((ref_6.i1 > ref_6.i2) OR (ref_5.i4 < ref_5.i4)))
+WHERE (t1.i2 >= t1.i2));
+i1
+drop table t1;
+#
+# MDEV-25761: Assertion `aggr != __null' failed in sub_select_postjoin_aggr
+#
+CREATE TABLE t1 ( a int NOT NULL PRIMARY KEY) engine=innodb;
+INSERT INTO t1 VALUES (0),(4),(31);
+CREATE TABLE t2 (i int) engine=innodb;
+DELETE FROM t1 WHERE t1.a =
+(SELECT t1.a FROM t2 UNION SELECT DISTINCT 52 FROM t2 r WHERE t1.a = t1.a);
+DROP TABLE t1,t2;

mysql-test/main/union_innodb.test

@@ -0,0 +1,45 @@
+--source include/have_innodb.inc
+ 
+--echo #
+--echo # MDEV-25636: Bug report: abortion in sql/sql_parse.cc:6294
+--echo #
+
+CREATE TABLE t1 (i1 int)engine=innodb;
+INSERT INTO `t1` VALUES (62),(66);
+CREATE TABLE t2 (i1 int) engine=innodb;
+ 
+SELECT 1 FROM t1 
+WHERE t1.i1 =( SELECT t1.i1 FROM t2  
+	UNION SELECT i1 FROM (t1 AS dt1 natural JOIN t2) 
+	window w1 as (partition by t1.i1));
+
+drop table t1,t2;
+
+--echo # Another testcase
+CREATE TABLE t1 (i3 int NOT NULL, i1 int , i2 int , i4 int , PRIMARY key(i2));
+INSERT INTO t1 VALUES (6,72,98,98),(46,1,6952,0);
+ 
+SELECT i1 FROM t1 
+WHERE t1.i3 =
+  (SELECT ref_4.i2 FROM t1 AS ref_4
+   	WHERE t1.i2 > (SELECT i3 FROM t1 ORDER BY i3 LIMIT 1 OFFSET 4)
+   UNION 
+   SELECT ref_6.i2
+   FROM (t1 AS ref_5 JOIN t1 AS ref_6 ON ((ref_6.i1 > ref_6.i2) OR (ref_5.i4 < ref_5.i4)))
+   WHERE (t1.i2 >= t1.i2));
+
+drop table t1;
+
+--echo #
+--echo # MDEV-25761: Assertion `aggr != __null' failed in sub_select_postjoin_aggr
+--echo #
+
+CREATE TABLE t1 ( a int NOT NULL PRIMARY KEY) engine=innodb;
+INSERT INTO t1 VALUES (0),(4),(31);
+ 
+CREATE TABLE t2 (i int) engine=innodb;
+ 
+DELETE FROM t1 WHERE t1.a =
+  (SELECT t1.a FROM t2 UNION SELECT DISTINCT 52 FROM t2 r WHERE t1.a = t1.a);
+
+DROP TABLE t1,t2;

mysql-test/main/win.result

@@ -4277,6 +4277,38 @@ drop procedure sp7;
 drop view v1,v2;
 drop table t1;
 #
+# MDEV-17785: Window functions not working in ONLY_FULL_GROUP_BY mode
+#
+CREATE TABLE t1(a VARCHAR(10), b int);
+INSERT INTO t1 VALUES
+('Maths', 60),('Maths', 60),
+('Maths', 70),('Maths', 55),
+('Biology', 60), ('Biology', 70);
+SET @save_sql_mode= @@sql_mode;
+SET sql_mode = 'ONLY_FULL_GROUP_BY';
+SELECT
+RANK() OVER (PARTITION BY a ORDER BY b) AS rank,
+a, b FROM t1 ORDER BY a, b DESC;
+rank	a	b
+2	Biology	70
+1	Biology	60
+4	Maths	70
+2	Maths	60
+2	Maths	60
+1	Maths	55
+SET sql_mode= @save_sql_mode;
+DROP TABLE t1;
+CREATE TABLE t1(i int,j int);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+SELECT i, LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) FROM t1 GROUP BY i;
+i	LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j)
+1	6
+2	4
+3	2
+4	2
+DROP TABLE t1;
+#
 # End of 10.2 tests
 #
 #

mysql-test/main/win.test

@@ -2702,6 +2702,33 @@ drop procedure sp7;
 drop view v1,v2;
 drop table t1;
 
+--echo #
+--echo # MDEV-17785: Window functions not working in ONLY_FULL_GROUP_BY mode
+--echo #
+
+CREATE TABLE t1(a VARCHAR(10), b int);
+
+INSERT INTO t1 VALUES
+('Maths', 60),('Maths', 60),
+('Maths', 70),('Maths', 55),
+('Biology', 60), ('Biology', 70);
+
+SET @save_sql_mode= @@sql_mode;
+SET sql_mode = 'ONLY_FULL_GROUP_BY';
+
+SELECT
+  RANK() OVER (PARTITION BY a ORDER BY b) AS rank,
+  a, b FROM t1 ORDER BY a, b DESC;
+
+SET sql_mode= @save_sql_mode;
+DROP TABLE t1;
+
+CREATE TABLE t1(i int,j int);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+SELECT i, LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) FROM t1 GROUP BY i;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/mariadb-test-run.pl

@@ -1783,7 +1783,7 @@ sub collect_mysqld_features_from_running_server ()
   }
 
   mtr_add_arg($args, "--silent"); # Tab separated output
-  mtr_add_arg($args, "-e '%s'", "use mysql; SHOW VARIABLES");
+  mtr_add_arg($args, "-e \"use mysql; SHOW VARIABLES\"");
   my $cmd= "$mysql " . join(' ', @$args);
   mtr_verbose("cmd: $cmd");
 

mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result

@@ -20,7 +20,7 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint/ in mysqld.1.err
 # empty redo log from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -28,7 +28,7 @@ COUNT(*)
 1
 FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Corrupted multi-file redo log from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -36,7 +36,7 @@ COUNT(*)
 0
 FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err
 # Empty multi-file redo log (wrong offset) from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -45,7 +45,7 @@ COUNT(*)
 FOUND 3 /Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint\./ in mysqld.1.err
 # Multi-file redo log with size mismatch from after MariaDB 10.2.2
 # Corrupted multi-file redo log from after MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -53,15 +53,16 @@ COUNT(*)
 0
 FOUND 3 /Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint\./ in mysqld.1.err
 FOUND 1 /InnoDB: No valid checkpoint was found; the log was created with BogoDB 1\.2\.3\.4\./ in mysqld.1.err
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 COUNT(*)
 0
+FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 1048576 bytes!/ in mysqld.1.err
 FOUND 2 /InnoDB: No valid checkpoint was found; the log was created with BogoDB 1\.2\.3\.4\./ in mysqld.1.err
 # Empty multi-file redo log from after MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -74,6 +75,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
+FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 4194304 bytes!/ in mysqld.1.err
 FOUND 1 /InnoDB: Invalid log header checksum/ in mysqld.1.err
 # distant future redo log format, with valid header checksum
 # restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption
@@ -165,7 +167,7 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND 3 /\[ERROR\] InnoDB: Upgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1\./ in mysqld.1.err
 # Empty 10.3 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -173,7 +175,7 @@ COUNT(*)
 1
 FOUND 1 /InnoDB: log sequence number 1213964\b.*; transaction id 0/ in mysqld.1.err
 # Empty 10.2 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -181,7 +183,7 @@ COUNT(*)
 1
 FOUND 3 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Empty 10.5 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');

mysql-test/suite/encryption/r/tempfiles_encrypted.result

@@ -4283,6 +4283,38 @@ drop procedure sp7;
 drop view v1,v2;
 drop table t1;
 #
+# MDEV-17785: Window functions not working in ONLY_FULL_GROUP_BY mode
+#
+CREATE TABLE t1(a VARCHAR(10), b int);
+INSERT INTO t1 VALUES
+('Maths', 60),('Maths', 60),
+('Maths', 70),('Maths', 55),
+('Biology', 60), ('Biology', 70);
+SET @save_sql_mode= @@sql_mode;
+SET sql_mode = 'ONLY_FULL_GROUP_BY';
+SELECT
+RANK() OVER (PARTITION BY a ORDER BY b) AS rank,
+a, b FROM t1 ORDER BY a, b DESC;
+rank	a	b
+2	Biology	70
+1	Biology	60
+4	Maths	70
+2	Maths	60
+2	Maths	60
+1	Maths	55
+SET sql_mode= @save_sql_mode;
+DROP TABLE t1;
+CREATE TABLE t1(i int,j int);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4);
+SELECT i, LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) FROM t1 GROUP BY i;
+i	LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j)
+1	6
+2	4
+3	2
+4	2
+DROP TABLE t1;
+#
 # End of 10.2 tests
 #
 #

mysql-test/suite/galera_sr/r/MDEV-27615.result

@@ -15,7 +15,7 @@ SET DEBUG_SYNC='now WAIT_FOR before_fragment';
 SET GLOBAL wsrep_cluster_address = '';
 SET DEBUG_SYNC = 'now SIGNAL continue';
 connection node_2;
-ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to server during query
 connection node_2a;
 SELECT * FROM mysql.wsrep_streaming_log;
 node_uuid	trx_id	seqno	flags	frag

mysql-test/suite/innodb/r/blob_cmp_empty.result

@@ -0,0 +1,19 @@
+connect prevent_purge,localhost,root,,;
+start transaction with consistent snapshot;
+connection default;
+SET @fill_amount = (@@innodb_page_size / 2 ) + 1;
+CREATE TABLE t1 (col_text TEXT NOT NULL, KEY (col_text(9))) ENGINE=InnoDB;
+INSERT INTO t1 (col_text) VALUES (REPEAT('x', @fill_amount));
+UPDATE t1 SET col_text='';
+UPDATE t1 SET col_text=REPEAT('y', @fill_amount);
+connect con1,localhost,root,,;
+SET @fill_amount = (@@innodb_page_size / 2 ) + 1;
+BEGIN;
+INSERT INTO t1 (col_text) VALUES (REPEAT('z', @fill_amount));
+connection default;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+disconnect con1;
+disconnect prevent_purge;
+DROP TABLE t1;

mysql-test/suite/innodb/r/ibuf_not_empty.result

@@ -5,6 +5,7 @@ c INT,
 INDEX(b))
 ENGINE=InnoDB STATS_PERSISTENT=0;
 SET GLOBAL innodb_change_buffering_debug = 1;
+SET GLOBAL innodb_change_buffering=all;
 INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_1024;
 # restart: --innodb-force-recovery=6 --innodb-change-buffer-dump
 check table t1;

mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result

@@ -13,6 +13,7 @@ c INT,
 INDEX(b))
 ENGINE=InnoDB STATS_PERSISTENT=0;
 SET GLOBAL innodb_change_buffering_debug = 1;
+SET GLOBAL innodb_change_buffering = all;
 INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192;
 BEGIN;
 SELECT b FROM t1 LIMIT 3;

mysql-test/suite/innodb/r/log_corruption.result

@@ -20,7 +20,7 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint/ in mysqld.1.err
 # empty redo log from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -28,7 +28,7 @@ COUNT(*)
 1
 FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Corrupted multi-file redo log from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -36,7 +36,7 @@ COUNT(*)
 0
 FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err
 # Empty multi-file redo log (wrong offset) from before MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -45,7 +45,7 @@ COUNT(*)
 FOUND 3 /Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint\./ in mysqld.1.err
 # Multi-file redo log with size mismatch from after MariaDB 10.2.2
 # Corrupted multi-file redo log from after MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -53,15 +53,16 @@ COUNT(*)
 0
 FOUND 3 /Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint\./ in mysqld.1.err
 FOUND 1 /InnoDB: No valid checkpoint was found; the log was created with BogoDB 1\.2\.3\.4\./ in mysqld.1.err
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 COUNT(*)
 0
+FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 1048576 bytes!/ in mysqld.1.err
 FOUND 2 /InnoDB: No valid checkpoint was found; the log was created with BogoDB 1\.2\.3\.4\./ in mysqld.1.err
 # Empty multi-file redo log from after MariaDB 10.2.2
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -74,6 +75,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
+FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 4194304 bytes!/ in mysqld.1.err
 FOUND 1 /InnoDB: Invalid log header checksum/ in mysqld.1.err
 # distant future redo log format, with valid header checksum
 # restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption
@@ -165,7 +167,7 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND 3 /\[ERROR\] InnoDB: Upgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1\./ in mysqld.1.err
 # Empty 10.3 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -173,7 +175,7 @@ COUNT(*)
 1
 FOUND 1 /InnoDB: log sequence number 1213964\b.*; transaction id 0/ in mysqld.1.err
 # Empty 10.2 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
@@ -181,7 +183,7 @@ COUNT(*)
 1
 FOUND 3 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Empty 10.5 redo log
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=2m
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');

mysql-test/suite/innodb/r/log_file_size.result

@@ -1,5 +1,5 @@
 CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB;
-# restart: --innodb-log-file-size=2m
+# restart: --innodb-log-file-size=4m
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK

mysql-test/suite/innodb/t/blob_cmp_empty.test

@@ -0,0 +1,26 @@
+--source include/innodb_row_format.inc
+--source include/count_sessions.inc
+--connect(prevent_purge,localhost,root,,)
+start transaction with consistent snapshot;
+
+--connection default
+SET @fill_amount = (@@innodb_page_size / 2 ) + 1;
+CREATE TABLE t1 (col_text TEXT NOT NULL, KEY (col_text(9))) ENGINE=InnoDB;
+
+INSERT INTO t1 (col_text) VALUES (REPEAT('x', @fill_amount));
+UPDATE t1 SET col_text='';
+UPDATE t1 SET col_text=REPEAT('y', @fill_amount);
+
+--connect(con1,localhost,root,,)
+SET @fill_amount = (@@innodb_page_size / 2 ) + 1;
+BEGIN;
+INSERT INTO t1 (col_text) VALUES (REPEAT('z', @fill_amount));
+
+--connection default
+# If the bug is not fixed, CHECK TABLE will complain about wrong secondary index
+# rows count
+CHECK TABLE t1;
+--disconnect con1
+--disconnect prevent_purge
+DROP TABLE t1;
+--source include/wait_until_count_sessions.inc

mysql-test/suite/innodb/t/ibuf_not_empty.test

@@ -24,6 +24,7 @@ ENGINE=InnoDB STATS_PERSISTENT=0;
 # change buffering is possible, so that the change buffer will be used
 # whenever possible.
 SET GLOBAL innodb_change_buffering_debug = 1;
+SET GLOBAL innodb_change_buffering=all;
 
 # Create enough rows for the table, so that the change buffer will be
 # used for modifying the secondary index page. There must be multiple

mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test

@@ -33,6 +33,7 @@ ENGINE=InnoDB STATS_PERSISTENT=0;
 # change buffering is possible, so that the change buffer will be used
 # whenever possible.
 SET GLOBAL innodb_change_buffering_debug = 1;
+SET GLOBAL innodb_change_buffering = all;
 let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
 
 # Create enough rows for the table, so that the change buffer will be

mysql-test/suite/innodb/t/log_corruption.test

@@ -16,7 +16,7 @@ call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
 call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\\r?$");
 call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed");
 call mtr.add_suppression("InnoDB: Decrypting checkpoint failed");
-call mtr.add_suppression("InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 1048576 bytes!");
+call mtr.add_suppression("InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files (1048576|4194304) bytes!");
 --enable_query_log
 
 let bugdir= $MYSQLTEST_VARDIR/tmp/log_corruption;
@@ -161,7 +161,7 @@ die unless seek(OUT, 0x800, 0);
 print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 0, 0xb2a);
 close OUT or die;
 EOF
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -191,7 +191,7 @@ print OUT chr(0) x 2048;
 close OUT or die;
 EOF
 
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -210,7 +210,7 @@ print OUT chr(0) x 1536;
 close OUT or die;
 EOF
 
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -241,7 +241,7 @@ close OUT or die;
 EOF
 
 --echo # Corrupted multi-file redo log from after MariaDB 10.2.2
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -259,13 +259,14 @@ print OUT chr(0);
 close OUT or die;
 EOF
 
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 --source include/shutdown_mysqld.inc
 --let SEARCH_PATTERN=InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 1048576 bytes!
+--source include/search_pattern_in_file.inc
 --let SEARCH_PATTERN=InnoDB: No valid checkpoint was found; the log was created with BogoDB 1\\.2\\.3\\.4\\.
 --source include/search_pattern_in_file.inc
 
@@ -282,7 +283,7 @@ print OUT $_, pack("N", mycrc32($_, 0, $polynomial));
 close OUT or die;
 EOF
 
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -304,6 +305,8 @@ EOF
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
+--let SEARCH_PATTERN=InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files 4194304 bytes!
+--source include/search_pattern_in_file.inc
 let SEARCH_PATTERN=InnoDB: Invalid log header checksum;
 --source include/search_pattern_in_file.inc
 
@@ -351,6 +354,9 @@ die unless seek(OUT, 0x210, 0);
 print OUT pack("NNx[264]", 0, 0x80c);
 print OUT pack("NNx[212]N", 0x590dbaac, 0xfe922582, 0xc72d49c4);
 close OUT or die;
+die unless open OUT, ">", "$ENV{bugdir}/ib_logfile1";
+print OUT pack("x[4194304]");
+close OUT or die;
 EOF
 # Anything below innodb_force_recovery=6 must find a valid redo log.
 # Missing tablespace files are tolerated already with innodb_force_recovery=1.
@@ -533,7 +539,7 @@ print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2);
 close OUT or die;
 EOF
 
---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=2m
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'

mysql-test/suite/innodb/t/log_file_size.test

@@ -55,7 +55,7 @@ let $check_no_innodb=SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 
---let $restart_parameters= --innodb-log-file-size=2m
+--let $restart_parameters= --innodb-log-file-size=4m
 --source include/start_mysqld.inc
 
 let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;

mysql-test/suite/innodb_gis/disabled.def

@@ -12,3 +12,4 @@
 
 rtree_concurrent_srch : MDEV-15284 COUNT(*) mismatch
 rtree_recovery : MDEV-15284 COUNT(*) mismatch
+rtree_compress2 : MDEV-16269 CHECK TABLE reports wrong count

mysql-test/suite/mariabackup/innodb_redo_log_overwrite.opt

@@ -1 +1 @@
---loose-innodb-log-file-size=2097152
+--loose-innodb-log-file-size=4194304

mysql-test/suite/mariabackup/innodb_redo_overwrite.opt

@@ -1 +1 @@
---loose-innodb-log-file-size=2m
+--loose-innodb-log-file-size=4194304

mysql-test/suite/mariabackup/innodb_redo_overwrite.result

@@ -1,5 +1,4 @@
-CREATE TABLE t(i INT) ENGINE=INNODB;
+CREATE TABLE t ENGINE=INNODB SELECT seq%10 i FROM seq_0_to_204796;
-INSERT INTO t SELECT seq%10 FROM seq_0_to_51199;
 # xtrabackup backup
 FOUND 1 /Was only able to copy log from \d+ to \d+, not \d+; try increasing innodb_log_file_size\b/ in backup.log
 NOT FOUND /failed: redo log block checksum does not match/ in backup.log

mysql-test/suite/mariabackup/innodb_redo_overwrite.test

@@ -3,8 +3,7 @@
 --source include/have_debug.inc
 --source include/have_sequence.inc
 
-CREATE TABLE t(i INT) ENGINE=INNODB;
+CREATE TABLE t ENGINE=INNODB SELECT seq%10 i FROM seq_0_to_204796;
-INSERT INTO t SELECT seq%10 FROM seq_0_to_51199;
 
 --echo # xtrabackup backup
 --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup

mysql-test/suite/perfschema/r/sxlock_func.result

@@ -10,6 +10,7 @@ name
 wait/synch/rwlock/innodb/dict_operation_lock
 wait/synch/rwlock/innodb/fil_space_latch
 wait/synch/rwlock/innodb/lock_latch
+wait/synch/rwlock/innodb/log_latch
 wait/synch/rwlock/innodb/trx_i_s_cache_lock
 wait/synch/rwlock/innodb/trx_purge_latch
 TRUNCATE TABLE performance_schema.events_waits_history_long;
@@ -41,6 +42,7 @@ ORDER BY event_name;
 event_name
 wait/synch/rwlock/innodb/fil_space_latch
 wait/synch/rwlock/innodb/lock_latch
+wait/synch/rwlock/innodb/log_latch
 SELECT event_name FROM performance_schema.events_waits_history_long
 WHERE event_name = 'wait/synch/sxlock/innodb/index_tree_rw_lock'
 AND operation IN ('try_shared_lock','shared_lock') LIMIT 1;

mysql-test/suite/rpl/t/rpl_relay_max_extension.test

@@ -28,6 +28,8 @@
 #            showed
 #
 
+# MDEV-27721 rpl.rpl_relay_max_extension test is not FreeBSD-compatible
+--source include/linux.inc
 --source include/have_innodb.inc
 --source include/have_binlog_format_row.inc
 --let $rpl_topology=1->2

mysql-test/suite/sys_vars/r/innodb_change_buffering_basic.result

@@ -1,28 +1,28 @@
 SET @start_global_value = @@global.innodb_change_buffering;
 SELECT @start_global_value;
 @start_global_value
-all
+none
 Valid values are 'all', 'deletes', 'changes', 'inserts', 'none', 'purges'
 select @@global.innodb_change_buffering in ('all', 'deletes', 'changes', 'inserts', 'none', 'purges');
 @@global.innodb_change_buffering in ('all', 'deletes', 'changes', 'inserts', 'none', 'purges')
 1
 select @@global.innodb_change_buffering;
 @@global.innodb_change_buffering
-all
+none
 select @@session.innodb_change_buffering;
 ERROR HY000: Variable 'innodb_change_buffering' is a GLOBAL variable
 show global variables like 'innodb_change_buffering';
 Variable_name	Value
-innodb_change_buffering	all
+innodb_change_buffering	none
 show session variables like 'innodb_change_buffering';
 Variable_name	Value
-innodb_change_buffering	all
+innodb_change_buffering	none
 select * from information_schema.global_variables where variable_name='innodb_change_buffering';
 VARIABLE_NAME	VARIABLE_VALUE
-INNODB_CHANGE_BUFFERING	all
+INNODB_CHANGE_BUFFERING	none
 select * from information_schema.session_variables where variable_name='innodb_change_buffering';
 VARIABLE_NAME	VARIABLE_VALUE
-INNODB_CHANGE_BUFFERING	all
+INNODB_CHANGE_BUFFERING	none
 set global innodb_change_buffering='none';
 select @@global.innodb_change_buffering;
 @@global.innodb_change_buffering
@@ -62,4 +62,4 @@ ERROR 42000: Variable 'innodb_change_buffering' can't be set to the value of 'so
 SET @@global.innodb_change_buffering = @start_global_value;
 SELECT @@global.innodb_change_buffering;
 @@global.innodb_change_buffering
-all
+none

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -227,7 +227,7 @@ READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	INNODB_CHANGE_BUFFERING
 SESSION_VALUE	NULL
-DEFAULT_VALUE	all
+DEFAULT_VALUE	none
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	ENUM
 VARIABLE_COMMENT	Buffer changes to secondary indexes.
@@ -1011,7 +1011,7 @@ DEFAULT_VALUE	100663296
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Redo log size in bytes.
-NUMERIC_MIN_VALUE	1048576
+NUMERIC_MIN_VALUE	4194304
 NUMERIC_MAX_VALUE	18446744073709551615
 NUMERIC_BLOCK_SIZE	4096
 ENUM_VALUE_LIST	NULL

mysys/my_gethwaddr.c

@@ -23,6 +23,7 @@
 
 #ifndef MAIN
 
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__) || defined(__sun) || defined(_WIN32)
 static my_bool memcpy_and_test(uchar *to, uchar *from, uint len)
 {
   uint i, res= 1;
@@ -32,6 +33,7 @@ static my_bool memcpy_and_test(uchar *to, uchar *from, uint len)
       res= 0;
   return res;
 }
+#endif
 
 #if defined(__APPLE__) || defined(__FreeBSD__)
 #include <net/ethernet.h>
@@ -195,4 +197,3 @@ int main(int argc __attribute__((unused)),char **argv)
   return 0;
 }
 #endif
-

scripts/wsrep_sst_mariabackup.sh

@@ -1,4 +1,7 @@
-#!/bin/bash -ue
+#!/usr/bin/env bash
+
+set -ue
+
 # Copyright (C) 2017-2022 MariaDB
 # Copyright (C) 2013 Percona Inc
 #

scripts/wsrep_sst_mysqldump.sh

@@ -1,4 +1,7 @@
-#!/bin/bash -ue
+#!/usr/bin/env bash
+
+set -ue
+
 # Copyright (C) 2009-2015 Codership Oy
 # Copyright (C) 2017-2021 MariaDB
 #

scripts/wsrep_sst_rsync.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -ue
+#!/usr/bin/env bash
+
+set -ue
 
 # Copyright (C) 2017-2022 MariaDB
 # Copyright (C) 2010-2014 Codership Oy
@@ -738,7 +740,7 @@ EOF
             elif [ "$OS" = 'Linux' ]; then
                tmpfile=$(mktemp "--tmpdir=$tmpdir")
             else
-               tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d')
+               tmpfile=$(TMPDIR="$tmpdir"; mktemp)
             fi
 
             wsrep_log_info "Extracting binlog files:"

sql/backup.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2018, 2021, MariaDB Corporation.
+/* Copyright (c) 2018, 2022, MariaDB Corporation.
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; version 2 of the License.
@@ -257,9 +257,13 @@ static bool backup_flush(THD *thd)
     This will probably require a callback from the InnoDB code.
 */
 
+/* Retry to get inital lock for 0.1 + 0.5 + 2.25 + 11.25 + 56.25 = 70.35 sec */
+#define MAX_RETRY_COUNT 5
+
 static bool backup_block_ddl(THD *thd)
 {
   PSI_stage_info org_stage;
+  uint sleep_time;
   DBUG_ENTER("backup_block_ddl");
 
   kill_delayed_threads();
@@ -302,11 +306,22 @@ static bool backup_block_ddl(THD *thd)
     block new DDL's, in addition to all previous blocks
     We didn't do this lock above, as we wanted DDL's to be executed while
     we wait for non transactional tables (which may take a while).
+
+    We do this lock in a loop as we can get a deadlock if there are multi-object
+    ddl statements like
+    RENAME TABLE t1 TO t2, t3 TO t3
+    and the MDL happens in the middle of it.
  */
   THD_STAGE_INFO(thd, stage_waiting_for_ddl);
-  if (thd->mdl_context.upgrade_shared_lock(backup_flush_ticket,
+  sleep_time= 100;                              // Start with 0.1 seconds
+  for (uint i= 0 ; i <= MAX_RETRY_COUNT ; i++)
+  {
+    if (!thd->mdl_context.upgrade_shared_lock(backup_flush_ticket,
                                               MDL_BACKUP_WAIT_DDL,
                                               thd->variables.lock_wait_timeout))
+      break;
+    if (thd->get_stmt_da()->sql_errno() != ER_LOCK_DEADLOCK || thd->killed ||
+        i == MAX_RETRY_COUNT)
     {
       /*
         Could be a timeout. Downgrade lock to what is was before this function
@@ -315,6 +330,10 @@ static bool backup_block_ddl(THD *thd)
       backup_flush_ticket->downgrade_lock(MDL_BACKUP_FLUSH);
       goto err;
     }
+    thd->clear_error();                         // Forget the DEADLOCK error
+    my_sleep(sleep_time);
+    sleep_time*= 5;                             // Wait a bit longer next time
+  }
 
   /* There can't be anything more that needs to be logged to ddl log */
   THD_STAGE_INFO(thd, org_stage);

sql/item_windowfunc.cc

@@ -111,6 +111,8 @@ Item_window_func::fix_fields(THD *thd, Item **ref)
     return true;
   }
 
+  window_func()->mark_as_window_func_sum_expr();
+
   /*
     TODO: why the last parameter is 'ref' in this call? What if window_func
     decides to substitute itself for something else and does *ref=.... ? 

sql/opt_range.cc

@@ -14112,7 +14112,7 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time)
         key_part_range[1]= last_part;
 
         /* Check if cur_part is referenced in the WHERE clause. */
-        if (join->conds->walk(&Item::find_item_in_field_list_processor, 0,
+        if (join->conds->walk(&Item::find_item_in_field_list_processor, true,
                               key_part_range))
         {
           cause= "keypart reference from where clause";

sql/records.cc

@@ -355,7 +355,7 @@ void end_read_record(READ_RECORD *info)
   free_cache(info);
   if (info->table)
   {
-    if (info->table->is_created())
+    if (info->table->db_stat) // if opened
       (void) info->table->file->extra(HA_EXTRA_NO_CACHE);
     if (info->read_record_func != rr_quick) // otherwise quick_range does it
       (void) info->table->file->ha_index_or_rnd_end();

sql/sql_base.cc

@@ -6457,8 +6457,11 @@ find_field_in_tables(THD *thd, Item_ident *item,
         for (SELECT_LEX *sl= current_sel; sl && sl!=last_select;
              sl=sl->outer_select())
         {
-          Item_in_subselect *in_subs=
+          Item *subs= sl->master_unit()->item;
-            sl->master_unit()->item->get_IN_subquery();
+          if (!subs)
+            continue;
+
+          Item_in_subselect *in_subs= subs->get_IN_subquery();
           if (in_subs &&
               in_subs->substype() == Item_subselect::IN_SUBS &&
               in_subs->test_strategy(SUBS_SEMI_JOIN))

sql/sql_lex.cc

@@ -4965,7 +4965,21 @@ bool st_select_lex::optimize_unflattened_subqueries(bool const_only)
       }
       if (empty_union_result)
         subquery_predicate->no_rows_in_result();
-      if (!is_correlated_unit)
+
+      if (is_correlated_unit)
+      {
+        /*
+          Some parts of UNION are not correlated. This means we will need to
+          re-execute the whole UNION every time. Mark all parts of the UNION
+          as correlated so that they are prepared to be executed multiple
+          times (if we don't do that, some part of the UNION may free its
+          execution data at the end of first execution and crash on the second
+          execution)
+        */
+        for (SELECT_LEX *sl= un->first_select(); sl; sl= sl->next_select())
+          sl->uncacheable |= UNCACHEABLE_DEPENDENT;
+      }
+      else
         un->uncacheable&= ~UNCACHEABLE_DEPENDENT;
       subquery_predicate->is_correlated= is_correlated_unit;
     }

sql/sql_select.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2016, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2021, MariaDB Corporation.
+   Copyright (c) 2009, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1728,7 +1728,8 @@ bool JOIN::prepare_stage2()
 #endif
   if (select_lex->olap == ROLLUP_TYPE && rollup_init())
     goto err;
-  if (alloc_func_list())
+  if (alloc_func_list() ||
+      make_sum_func_list(all_fields, fields_list, false))
     goto err;
 
   res= FALSE;
@@ -2359,7 +2360,21 @@ JOIN::optimize_inner()
       If all items were resolved by opt_sum_query, there is no need to
       open any tables.
     */
-    if ((res=opt_sum_query(thd, select_lex->leaf_tables, all_fields, conds)))
+
+    /*
+      The following resetting and restoring of sum_funcs is needed to
+      go around a bug in spider where it assumes that
+      make_sum_func_list() has not been called yet and do logical
+      choices based on this if special handling of min/max functions should
+      be done. We disable this special handling while we are trying to find
+      out if we can replace MIN/MAX values with constants.
+    */
+    Item_sum **save_func_sums= sum_funcs, *tmp_sum_funcs= 0;
+    sum_funcs= &tmp_sum_funcs;
+    res= opt_sum_query(thd, select_lex->leaf_tables, all_fields, conds);
+    sum_funcs= save_func_sums;
+
+    if (res)
     {
       DBUG_ASSERT(res >= 0);
       if (res == HA_ERR_KEY_NOT_FOUND)
@@ -2957,18 +2972,14 @@ int JOIN::optimize_stage2()
   }
 
   /*
-    Remove ORDER BY in the following cases:
+    We can ignore ORDER BY if it's a prefix of the GROUP BY list
-    - GROUP BY is more specific. Example GROUP BY a, b ORDER BY a
+    (as MariaDB is by default sorting on GROUP BY) or
-    - If there are aggregate functions and no GROUP BY, this always leads
+    if there is no GROUP BY and aggregate functions are used
-      to one row result, no point in sorting.
+    (as the result will only contain one row).
   */
-  if (test_if_subpart(group_list, order) ||
+  if (order && (test_if_subpart(group_list, order) ||
-      (!group_list && tmp_table_param.sum_func_count))
+                (!group_list && tmp_table_param.sum_func_count)))
-  {
+    order=0;
-    order= 0;
-    if (is_indexed_agg_distinct(this, NULL))
-      sort_and_group= 0;
-  }
 
   // Can't use sort on head table if using join buffering
   if (full_join || hash_join)
@@ -3000,7 +3011,6 @@ int JOIN::optimize_stage2()
   if (select_lex->have_window_funcs())
     simple_order= FALSE;
 
-
   /*
     If the hint FORCE INDEX FOR ORDER BY/GROUP BY is used for the table
     whose columns are required to be returned in a sorted order, then
@@ -3733,7 +3743,7 @@ bool JOIN::make_aggr_tables_info()
       // for the first table
       if (group_list || tmp_table_param.sum_func_count)
       {
-        if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true, true))
+        if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true))
           DBUG_RETURN(true);
         if (prepare_sum_aggregators(thd, sum_funcs,
                                     !join_tab->is_using_agg_loose_index_scan()))
@@ -3843,7 +3853,7 @@ bool JOIN::make_aggr_tables_info()
       last_tab->all_fields= &tmp_all_fields3;
       last_tab->fields= &tmp_fields_list3;
     }
-    if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true, true))
+    if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true))
       DBUG_RETURN(true);
     if (prepare_sum_aggregators(thd, sum_funcs,
                                 !join_tab ||
@@ -4060,8 +4070,6 @@ JOIN::create_postjoin_aggr_table(JOIN_TAB *tab, List<Item> *table_fields,
   }
   else
   {
-    if (make_sum_func_list(all_fields, fields_list, false))
-      goto err;
     if (prepare_sum_aggregators(thd, sum_funcs,
                                 !join_tab->is_using_agg_loose_index_scan()))
       goto err;
@@ -7314,7 +7322,6 @@ void optimize_keyuse(JOIN *join, DYNAMIC_ARRAY *keyuse_array)
   Check for the presence of AGGFN(DISTINCT a) queries that may be subject
   to loose index scan.
 
-
   Check if the query is a subject to AGGFN(DISTINCT) using loose index scan
   (QUICK_GROUP_MIN_MAX_SELECT).
   Optionally (if out_args is supplied) will push the arguments of 
@@ -7353,9 +7360,6 @@ is_indexed_agg_distinct(JOIN *join, List<Item_field> *out_args)
       join->select_lex->olap == ROLLUP_TYPE)  /* Check (B3) for ROLLUP */
     return false;
 
-  if (join->make_sum_func_list(join->all_fields, join->fields_list, true))
-    return false;
-
   Bitmap<MAX_FIELDS> first_aggdistinct_fields;
   bool first_aggdistinct_fields_initialized= false;
   for (sum_item_ptr= join->sum_funcs; *sum_item_ptr; sum_item_ptr++)
@@ -7457,16 +7461,23 @@ add_group_and_distinct_keys(JOIN *join, JOIN_TAB *join_tab)
     while ((item= select_items_it++))
       item->walk(&Item::collect_item_field_processor, 0, &indexed_fields);
   }
-  else if (join->tmp_table_param.sum_func_count &&
+  else if (!join->tmp_table_param.sum_func_count ||
-           is_indexed_agg_distinct(join, &indexed_fields))
+           !is_indexed_agg_distinct(join, &indexed_fields))
   {
-    join->sort_and_group= 1;
+    /*
-  }
+      There where no GROUP BY fields and also either no aggregate
-  else
+      functions or not all aggregate functions where used with the
+      same DISTINCT (or MIN() / MAX() that works similarly).
+      Nothing to do there.
+    */
     return;
+  }
 
   if (indexed_fields.elements == 0)
+  {
+    /* There where no index we could use to satisfy the GROUP BY */
     return;
+  }
 
   /* Intersect the keys of all group fields. */
   cur_item= indexed_fields_it++;
@@ -25951,16 +25962,13 @@ bool JOIN::alloc_func_list()
 
 bool JOIN::make_sum_func_list(List<Item> &field_list,
                               List<Item> &send_result_set_metadata,
-			      bool before_group_by, bool recompute)
+			      bool before_group_by)
 {
   List_iterator_fast<Item> it(field_list);
   Item_sum **func;
   Item *item;
   DBUG_ENTER("make_sum_func_list");
 
-  if (*sum_funcs && !recompute)
-    DBUG_RETURN(FALSE); /* We have already initialized sum_funcs. */
-
   func= sum_funcs;
   while ((item=it++))
   {
@@ -26107,7 +26115,7 @@ change_to_use_tmp_fields(THD *thd, Ref_ptr_array ref_pointer_array,
   Change all funcs to be fields in tmp table.
 
   @param thd                   THD pointer
-  @param ref_pointer_array     array of pointers to top elements of filed list
+  @param ref_pointer_array     array of pointers to top elements of field list
   @param res_selected_fields   new list of items of select item list
   @param res_all_fields        new list of all items
   @param elements              number of elements in select item list

sql/sql_select.h

@@ -1206,7 +1206,17 @@ public:
     Indicates that grouping will be performed on the result set during
     query execution. This field belongs to query execution.
 
-    @see make_group_fields, alloc_group_fields, JOIN::exec
+    If 'sort_and_group' is set, then the optimizer is going to use on of
+    the following algorithms to resolve GROUP BY.
+
+    - If one table, sort the table and then calculate groups on the fly.
+    - If more than one table, create a temporary table to hold the join,
+      sort it and then resolve group by on the fly.
+
+    The 'on the fly' calculation is done in end_send_group()
+
+    @see make_group_fields, alloc_group_fields, JOIN::exec,
+         setup_end_select_func
   */
   bool     sort_and_group; 
   bool     first_record,full_join, no_field_update;
@@ -1585,7 +1595,7 @@ public:
   bool make_range_rowid_filters();
   bool init_range_rowid_filters();
   bool make_sum_func_list(List<Item> &all_fields, List<Item> &send_fields,
-			  bool before_group_by, bool recompute= FALSE);
+			  bool before_group_by);
 
   /// Initialzes a slice, see comments for ref_ptrs above.
   Ref_ptr_array ref_ptr_array_slice(size_t slice_num)

sql/sql_yacc.yy

@@ -10724,9 +10724,6 @@ window_func:
           simple_window_func
         |
           sum_expr
-          {
-            ((Item_sum *) $1)->mark_as_window_func_sum_expr();
-          }
         |
           function_call_generic
           {
@@ -10739,8 +10736,6 @@ window_func:
               thd->parse_error();
               MYSQL_YYABORT;
             }
-
-            ((Item_sum *) $1)->mark_as_window_func_sum_expr();
           }
         ;
 

storage/innobase/buf/buf0flu.cc

@@ -115,6 +115,7 @@ static void buf_flush_validate_skip()
 /** Wake up the page cleaner if needed */
 void buf_pool_t::page_cleaner_wakeup()
 {
+  ut_d(buf_flush_validate_skip());
   if (!page_cleaner_idle())
     return;
   double dirty_pct= double(UT_LIST_GET_LEN(buf_pool.flush_list)) * 100.0 /
@@ -155,7 +156,7 @@ void buf_pool_t::page_cleaner_wakeup()
   }
 }
 
-inline void buf_pool_t::delete_from_flush_list_low(buf_page_t *bpage)
+inline void buf_pool_t::delete_from_flush_list_low(buf_page_t *bpage) noexcept
 {
   ut_ad(!fsp_is_system_temporary(bpage->id().space()));
   mysql_mutex_assert_owner(&flush_list_mutex);
@@ -166,13 +167,9 @@ inline void buf_pool_t::delete_from_flush_list_low(buf_page_t *bpage)
 /** Insert a modified block into the flush list.
 @param block    modified block
 @param lsn      start LSN of the mini-transaction that modified the block */
-void buf_pool_t::insert_into_flush_list(buf_block_t *block, lsn_t lsn)
+void buf_pool_t::insert_into_flush_list(buf_block_t *block, lsn_t lsn) noexcept
 {
-  mysql_mutex_assert_not_owner(&mutex);
+  ut_ad(recv_recovery_is_on() || log_sys.latch.is_locked());
-#ifdef SAFE_MUTEX
-  if (!recv_recovery_is_on())
-    mysql_mutex_assert_owner(&log_sys.flush_order_mutex);
-#endif /* SAFE_MUTEX */
   ut_ad(lsn > 2);
   static_assert(log_t::FIRST_LSN >= 2, "compatibility");
   ut_ad(!fsp_is_system_temporary(block->page.id().space()));
@@ -191,16 +188,27 @@ void buf_pool_t::insert_into_flush_list(buf_block_t *block, lsn_t lsn)
   MEM_CHECK_DEFINED(block->page.zip.data
                     ? block->page.zip.data : block->page.frame,
                     block->physical_size());
+  if (buf_page_t *prev= UT_LIST_GET_FIRST(flush_list))
+  {
+    if (prev->oldest_modification() <= lsn)
+      goto insert_first;
+    while (buf_page_t *next= UT_LIST_GET_NEXT(list, prev))
+      if (next->oldest_modification() <= lsn)
+        break;
+      else
+        prev= next;
+    UT_LIST_INSERT_AFTER(flush_list, prev, &block->page);
+  }
+  else
+  insert_first:
     UT_LIST_ADD_FIRST(flush_list, &block->page);
-  ut_d(buf_flush_validate_skip());
-  page_cleaner_wakeup();
   mysql_mutex_unlock(&flush_list_mutex);
 }
 
 /** Remove a block from flush_list.
 @param bpage   buffer pool page
 @param clear   whether to invoke buf_page_t::clear_oldest_modification() */
-void buf_pool_t::delete_from_flush_list(buf_page_t *bpage, bool clear)
+void buf_pool_t::delete_from_flush_list(buf_page_t *bpage, bool clear) noexcept
 {
   delete_from_flush_list_low(bpage);
   stat.flush_list_bytes-= bpage->physical_size();
@@ -743,7 +751,7 @@ not_compressed:
 }
 
 /** Free a page whose underlying file page has been freed. */
-inline void buf_pool_t::release_freed_page(buf_page_t *bpage)
+inline void buf_pool_t::release_freed_page(buf_page_t *bpage) noexcept
 {
   mysql_mutex_assert_owner(&mutex);
   mysql_mutex_lock(&flush_list_mutex);
@@ -1696,12 +1704,12 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept
 #endif
   {
     n_pending_checkpoint_writes++;
-    mysql_mutex_unlock(&mutex);
+    latch.wr_unlock();
     /* FIXME: issue an asynchronous write */
     log.write(offset, {c, get_block_size()});
     if (srv_file_flush_method != SRV_O_DSYNC)
       ut_a(log.flush());
-    mysql_mutex_lock(&mutex);
+    latch.wr_lock(SRW_LOCK_CALL);
     n_pending_checkpoint_writes--;
   }
 
@@ -1712,7 +1720,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept
   DBUG_PRINT("ib_log", ("checkpoint ended at " LSN_PF ", flushed to " LSN_PF,
                         next_checkpoint_lsn, get_flushed_lsn()));
 
-  mysql_mutex_unlock(&mutex);
+  latch.wr_unlock();
 }
 
 /** Initiate a log checkpoint, discarding the start of the log.
@@ -1722,7 +1730,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept
 static bool log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn)
 {
   ut_ad(!srv_read_only_mode);
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_write_locked());
   ut_ad(oldest_lsn <= end_lsn);
   ut_ad(end_lsn == log_sys.get_lsn());
   ut_ad(!recv_no_log_write);
@@ -1735,7 +1743,7 @@ static bool log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn)
     /* Do nothing, because nothing was logged (other than a
     FILE_CHECKPOINT record) since the previous checkpoint. */
   do_nothing:
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.wr_unlock();
     return true;
   }
 
@@ -1748,13 +1756,14 @@ static bool log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn)
 
   It is important that we write out the redo log before any further
   dirty pages are flushed to the tablespace files.  At this point,
-  because we hold log_sys.mutex, mtr_t::commit() in other threads will
+  because we hold exclusive log_sys.latch,
-  be blocked, and no pages can be added to the flush lists. */
+  mtr_t::commit() in other threads will be blocked,
+  and no pages can be added to buf_pool.flush_list. */
   const lsn_t flush_lsn{fil_names_clear(oldest_lsn)};
   ut_ad(flush_lsn >= end_lsn + SIZE_OF_FILE_CHECKPOINT);
-  mysql_mutex_unlock(&log_sys.mutex);
+  log_sys.latch.wr_unlock();
   log_write_up_to(flush_lsn, true);
-  mysql_mutex_lock(&log_sys.mutex);
+  log_sys.latch.wr_lock(SRW_LOCK_CALL);
   if (log_sys.last_checkpoint_lsn >= oldest_lsn)
     goto do_nothing;
 
@@ -1763,13 +1772,12 @@ static bool log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn)
   if (log_sys.n_pending_checkpoint_writes)
   {
     /* A checkpoint write is running */
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.wr_unlock();
     return false;
   }
 
   log_sys.next_checkpoint_lsn= oldest_lsn;
   log_sys.write_checkpoint(end_lsn);
-  mysql_mutex_assert_not_owner(&log_sys.mutex);
 
   return true;
 }
@@ -1793,12 +1801,10 @@ static bool log_checkpoint()
     fil_flush_file_spaces();
   }
 
-  mysql_mutex_lock(&log_sys.mutex);
+  log_sys.latch.wr_lock(SRW_LOCK_CALL);
   const lsn_t end_lsn= log_sys.get_lsn();
-  mysql_mutex_lock(&log_sys.flush_order_mutex);
   mysql_mutex_lock(&buf_pool.flush_list_mutex);
   const lsn_t oldest_lsn= buf_pool.get_oldest_modification(end_lsn);
-  mysql_mutex_unlock(&log_sys.flush_order_mutex);
   mysql_mutex_unlock(&buf_pool.flush_list_mutex);
   return log_checkpoint_low(oldest_lsn, end_lsn);
 }
@@ -1835,7 +1841,6 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
 {
   ut_ad(sync_lsn);
   ut_ad(sync_lsn < LSN_MAX);
-  mysql_mutex_assert_not_owner(&log_sys.mutex);
   ut_ad(!srv_read_only_mode);
 
   if (recv_recovery_is_on())
@@ -1893,7 +1898,6 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
 @param furious  true=furious flushing, false=limit to innodb_io_capacity */
 ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious)
 {
-  mysql_mutex_assert_not_owner(&log_sys.mutex);
   ut_ad(!srv_read_only_mode);
 
   if (recv_recovery_is_on())
@@ -1952,11 +1956,9 @@ ATTRIBUTE_COLD static void buf_flush_sync_for_checkpoint(lsn_t lsn)
       fil_flush_file_spaces();
     }
 
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
     const lsn_t newest_lsn= log_sys.get_lsn();
-    mysql_mutex_lock(&log_sys.flush_order_mutex);
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
-    mysql_mutex_unlock(&log_sys.flush_order_mutex);
     lsn_t measure= buf_pool.get_oldest_modification(0);
     const lsn_t checkpoint_lsn= measure ? measure : newest_lsn;
 
@@ -1970,13 +1972,11 @@ ATTRIBUTE_COLD static void buf_flush_sync_for_checkpoint(lsn_t lsn)
     }
     else
     {
-      mysql_mutex_unlock(&log_sys.mutex);
+      log_sys.latch.wr_unlock();
       if (!measure)
         measure= LSN_MAX;
     }
 
-    mysql_mutex_assert_not_owner(&log_sys.mutex);
-
     /* After attempting log checkpoint, check if we have reached our target. */
     const lsn_t target= buf_flush_sync_lsn;
 

storage/innobase/dict/dict0stats_bg.cc

@@ -103,17 +103,18 @@ static void dict_stats_recalc_pool_add(table_id_t id)
 {
   ut_ad(!srv_read_only_mode);
   ut_ad(id);
+  bool schedule = false;
   mysql_mutex_lock(&recalc_pool_mutex);
 
   const auto begin= recalc_pool.begin(), end= recalc_pool.end();
   if (end == std::find_if(begin, end, [&](const recalc &r){return r.id == id;}))
   {
     recalc_pool.emplace_back(recalc{id, recalc::IDLE});
+    schedule = true;
   }
 
   mysql_mutex_unlock(&recalc_pool_mutex);
-
+  if (schedule)
-  if (begin == end)
     dict_stats_schedule_now();
 }
 

storage/innobase/fil/fil0fil.cc

@@ -881,18 +881,21 @@ bool fil_space_free(uint32_t id, bool x_latched)
 		}
 
 		if (!recv_recovery_is_on()) {
-			mysql_mutex_lock(&log_sys.mutex);
+			log_sys.latch.wr_lock(SRW_LOCK_CALL);
-		}
 
-		mysql_mutex_assert_owner(&log_sys.mutex);
+			if (space->max_lsn) {
-
-		if (space->max_lsn != 0) {
 				ut_d(space->max_lsn = 0);
 				fil_system.named_spaces.remove(*space);
 			}
 
-		if (!recv_recovery_is_on()) {
+			log_sys.latch.wr_unlock();
-			mysql_mutex_unlock(&log_sys.mutex);
+		} else {
+			ut_ad(log_sys.latch.is_write_locked());
+
+			if (space->max_lsn) {
+				ut_d(space->max_lsn = 0);
+				fil_system.named_spaces.remove(*space);
+			}
 		}
 
 		fil_space_free_low(space);
@@ -1474,9 +1477,9 @@ static void fil_name_write_rename_low(uint32_t space_id, const char *old_name,
 
 static void fil_name_commit_durable(mtr_t *mtr)
 {
-  mysql_mutex_lock(&log_sys.mutex);
+  log_sys.latch.wr_lock(SRW_LOCK_CALL);
   auto lsn= mtr->commit_files();
-  mysql_mutex_unlock(&log_sys.mutex);
+  log_sys.latch.wr_unlock();
   log_write_up_to(lsn, true);
 }
 
@@ -1647,13 +1650,13 @@ pfs_os_file_t fil_delete_tablespace(uint32_t id)
     handle= fil_system.detach(space, true);
     mysql_mutex_unlock(&fil_system.mutex);
 
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
     if (space->max_lsn)
     {
       ut_d(space->max_lsn = 0);
       fil_system.named_spaces.remove(*space);
     }
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.wr_unlock();
 
     fil_space_free_low(space);
   }
@@ -1853,11 +1856,12 @@ static bool fil_rename_tablespace(uint32_t id, const char *old_path,
 	ut_ad(strchr(new_file_name, '/'));
 
 	if (!recv_recovery_is_on()) {
-		mysql_mutex_lock(&log_sys.mutex);
+		log_sys.latch.wr_lock(SRW_LOCK_CALL);
 	}
 
-	/* log_sys.mutex is above fil_system.mutex in the latching order */
+	/* log_sys.latch is above fil_system.mutex in the latching order */
-	mysql_mutex_assert_owner(&log_sys.mutex);
+	ut_ad(log_sys.latch.is_write_locked() ||
+	      srv_operation == SRV_OPERATION_RESTORE_DELTA);
 	mysql_mutex_lock(&fil_system.mutex);
 	space->release();
 	ut_ad(node->name == old_file_name);
@@ -1880,7 +1884,7 @@ skip_second_rename:
 	}
 
 	if (!recv_recovery_is_on()) {
-		mysql_mutex_unlock(&log_sys.mutex);
+		log_sys.latch.wr_unlock();
 	}
 
 	mysql_mutex_unlock(&fil_system.mutex);
@@ -3031,7 +3035,7 @@ void
 fil_names_dirty(
 	fil_space_t*	space)
 {
-	mysql_mutex_assert_owner(&log_sys.mutex);
+	ut_ad(log_sys.latch.is_write_locked());
 	ut_ad(recv_recovery_is_on());
 	ut_ad(log_sys.get_lsn() != 0);
 	ut_ad(space->max_lsn == 0);
@@ -3045,7 +3049,7 @@ fil_names_dirty(
 tablespace was modified for the first time since fil_names_clear(). */
 ATTRIBUTE_NOINLINE ATTRIBUTE_COLD void mtr_t::name_write()
 {
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_write_locked());
   ut_d(fil_space_validate_for_mtr_commit(m_user_space));
   ut_ad(!m_user_space->max_lsn);
   m_user_space->max_lsn= log_sys.get_lsn();
@@ -3073,7 +3077,7 @@ lsn_t fil_names_clear(lsn_t lsn)
 {
 	mtr_t	mtr;
 
-	mysql_mutex_assert_owner(&log_sys.mutex);
+	ut_ad(log_sys.latch.is_write_locked());
 	ut_ad(lsn);
 	ut_ad(log_sys.is_latest());
 

storage/innobase/handler/ha_innodb.cc

@@ -242,10 +242,10 @@ static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *,
     if (thd_kill_level(thd))
       break;
     /* Adjust for purge_coordinator_state::refresh() */
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.rd_lock(SRW_LOCK_CALL);
     const lsn_t last= log_sys.last_checkpoint_lsn,
       max_age= log_sys.max_checkpoint_age;
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.rd_unlock();
     const lsn_t lsn= log_sys.get_lsn();
     if ((lsn - last) / 4 >= max_age / 5)
       buf_flush_ahead(last + max_age / 5, false);
@@ -534,8 +534,6 @@ mysql_pfs_key_t	fts_pll_tokenize_mutex_key;
 mysql_pfs_key_t	ibuf_bitmap_mutex_key;
 mysql_pfs_key_t	ibuf_mutex_key;
 mysql_pfs_key_t	ibuf_pessimistic_insert_mutex_key;
-mysql_pfs_key_t	log_sys_mutex_key;
-mysql_pfs_key_t	log_flush_order_mutex_key;
 mysql_pfs_key_t	recalc_pool_mutex_key;
 mysql_pfs_key_t	purge_sys_pq_mutex_key;
 mysql_pfs_key_t	recv_sys_mutex_key;
@@ -571,12 +569,10 @@ static PSI_mutex_info all_innodb_mutexes[] = {
 	PSI_KEY(fts_cache_init_mutex),
 	PSI_KEY(fts_delete_mutex),
 	PSI_KEY(fts_doc_id_mutex),
-	PSI_KEY(log_flush_order_mutex),
 	PSI_KEY(ibuf_bitmap_mutex),
 	PSI_KEY(ibuf_mutex),
 	PSI_KEY(ibuf_pessimistic_insert_mutex),
 	PSI_KEY(index_online_log),
-	PSI_KEY(log_sys_mutex),
 	PSI_KEY(page_zip_stat_per_index_mutex),
 	PSI_KEY(purge_sys_pq_mutex),
 	PSI_KEY(recv_sys_mutex),
@@ -603,6 +599,7 @@ mysql_pfs_key_t	fil_space_latch_key;
 mysql_pfs_key_t trx_i_s_cache_lock_key;
 mysql_pfs_key_t	trx_purge_latch_key;
 mysql_pfs_key_t lock_latch_key;
+mysql_pfs_key_t	log_latch_key;
 
 /* all_innodb_rwlocks array contains rwlocks that are
 performance schema instrumented if "UNIV_PFS_RWLOCK"
@@ -617,6 +614,7 @@ static PSI_rwlock_info all_innodb_rwlocks[] =
   { &trx_i_s_cache_lock_key, "trx_i_s_cache_lock", 0 },
   { &trx_purge_latch_key, "trx_purge_latch", 0 },
   { &lock_latch_key, "lock_latch", 0 },
+  { &log_latch_key, "log_latch", 0 },
   { &index_tree_rw_lock_key, "index_tree_rw_lock", PSI_RWLOCK_FLAG_SX }
 };
 # endif /* UNIV_PFS_RWLOCK */
@@ -949,8 +947,7 @@ static SHOW_VAR innodb_status_variables[]= {
   {"buffer_pool_reads",
    &export_vars.innodb_buffer_pool_reads, SHOW_SIZE_T},
   {"buffer_pool_wait_free", &buf_pool.stat.LRU_waits, SHOW_SIZE_T},
-  {"buffer_pool_write_requests",
+  {"buffer_pool_write_requests", &buf_pool.flush_list_requests, SHOW_SIZE_T},
-   &export_vars.innodb_buffer_pool_write_requests, SHOW_SIZE_T},
   {"checkpoint_age", &export_vars.innodb_checkpoint_age, SHOW_SIZE_T},
   {"checkpoint_max_age", &export_vars.innodb_checkpoint_max_age, SHOW_SIZE_T},
   {"data_fsyncs", (size_t*) &os_n_fsyncs, SHOW_SIZE_T},
@@ -19235,7 +19232,7 @@ static MYSQL_SYSVAR_SIZE_T(log_buffer_size, log_sys.buf_size,
 static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
   "Redo log size in bytes.",
-  NULL, NULL, 96 << 20, 1U << 20, std::numeric_limits<ulonglong>::max(), 4096);
+  NULL, NULL, 96 << 20, 4 << 20, std::numeric_limits<ulonglong>::max(), 4096);
 
 static MYSQL_SYSVAR_UINT(old_blocks_pct, innobase_old_blocks_pct,
   PLUGIN_VAR_RQCMDARG,
@@ -19369,7 +19366,7 @@ static MYSQL_SYSVAR_BOOL(numa_interleave, srv_numa_interleave,
 static MYSQL_SYSVAR_ENUM(change_buffering, innodb_change_buffering,
   PLUGIN_VAR_RQCMDARG,
   "Buffer changes to secondary indexes.",
-  NULL, NULL, IBUF_USE_ALL, &innodb_change_buffering_typelib);
+  NULL, NULL, IBUF_USE_NONE, &innodb_change_buffering_typelib);
 
 static MYSQL_SYSVAR_UINT(change_buffer_max_size,
   srv_change_buffer_max_size,

storage/innobase/include/buf0buf.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2021, MariaDB Corporation.
+Copyright (c) 2013, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1755,6 +1755,9 @@ public:
   FlushHp flush_hp;
   /** modified blocks (a subset of LRU) */
   UT_LIST_BASE_NODE_T(buf_page_t) flush_list;
+  /** number of blocks ever added to flush_list;
+  protected by flush_list_mutex */
+  size_t flush_list_requests;
 private:
   /** whether the page cleaner needs wakeup from indefinite sleep */
   bool page_cleaner_is_idle;
@@ -1765,7 +1768,7 @@ public:
   pthread_cond_t do_flush_list;
 
   /** @return whether the page cleaner must sleep due to being idle */
-  bool page_cleaner_idle() const
+  bool page_cleaner_idle() const noexcept
   {
     mysql_mutex_assert_owner(&flush_list_mutex);
     return page_cleaner_is_idle;
@@ -1885,24 +1888,24 @@ public:
 
 private:
   /** Remove a block from the flush list. */
-  inline void delete_from_flush_list_low(buf_page_t *bpage);
+  inline void delete_from_flush_list_low(buf_page_t *bpage) noexcept;
   /** Remove a block from flush_list.
   @param bpage   buffer pool page
   @param clear   whether to invoke buf_page_t::clear_oldest_modification() */
-  void delete_from_flush_list(buf_page_t *bpage, bool clear);
+  void delete_from_flush_list(buf_page_t *bpage, bool clear) noexcept;
 public:
   /** Remove a block from flush_list.
   @param bpage   buffer pool page */
-  void delete_from_flush_list(buf_page_t *bpage)
+  void delete_from_flush_list(buf_page_t *bpage) noexcept
   { delete_from_flush_list(bpage, true); }
 
   /** Insert a modified block into the flush list.
   @param block    modified block
   @param lsn      start LSN of the mini-transaction that modified the block */
-  void insert_into_flush_list(buf_block_t *block, lsn_t lsn);
+  void insert_into_flush_list(buf_block_t *block, lsn_t lsn) noexcept;
 
   /** Free a page whose underlying file page has been freed. */
-  inline void release_freed_page(buf_page_t *bpage);
+  inline void release_freed_page(buf_page_t *bpage) noexcept;
 
 private:
   /** Temporary memory for page_compressed and encrypted I/O */

storage/innobase/include/buf0flu.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2021, MariaDB Corporation.
+Copyright (c) 2014, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -103,33 +103,6 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn);
 @param furious  true=furious flushing, false=limit to innodb_io_capacity */
 ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious);
 
-/********************************************************************//**
-This function should be called at a mini-transaction commit, if a page was
-modified in it. Puts the block to the list of modified blocks, if it not
-already in it. */
-inline void buf_flush_note_modification(buf_block_t *b, lsn_t start, lsn_t end)
-{
-  ut_ad(!srv_read_only_mode);
-  ut_d(const auto s= b->page.state());
-  ut_ad(s > buf_page_t::FREED);
-  ut_ad(s < buf_page_t::READ_FIX);
-  ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= end);
-  mach_write_to_8(b->page.frame + FIL_PAGE_LSN, end);
-  if (UNIV_LIKELY_NULL(b->page.zip.data))
-    memcpy_aligned<8>(FIL_PAGE_LSN + b->page.zip.data,
-                      FIL_PAGE_LSN + b->page.frame, 8);
-
-  const lsn_t oldest_modification= b->page.oldest_modification();
-
-  if (oldest_modification > 1)
-    ut_ad(oldest_modification <= start);
-  else if (fsp_is_system_temporary(b->page.id().space()))
-    b->page.set_temp_modified();
-  else
-    buf_pool.insert_into_flush_list(b, start);
-  srv_stats.buf_pool_write_requests.inc();
-}
-
 /** Initialize page_cleaner. */
 ATTRIBUTE_COLD void buf_flush_page_cleaner_init();
 

storage/innobase/include/fil0fil.h

@@ -354,13 +354,10 @@ struct fil_space_t final
 
   /** fil_system.spaces chain node */
   fil_space_t *hash;
+  /** LSN of the most recent fil_names_write_if_was_clean().
+  Reset to 0 by fil_names_clear(). Protected by exclusive log_sys.latch.
+  If and only if max_lsn is nonzero, this is in fil_system.named_spaces. */
   lsn_t max_lsn;
-				/*!< LSN of the most recent
-				fil_names_write_if_was_clean().
-				Reset to 0 by fil_names_clear().
-				Protected by log_sys.mutex.
-				If and only if this is nonzero, the
-				tablespace will be in named_spaces. */
   /** tablespace identifier */
   uint32_t id;
 	/** whether undo tablespace truncation is in progress */
@@ -1043,7 +1040,7 @@ struct fil_node_t final
 {
   /** tablespace containing this file */
   fil_space_t *space;
-  /** file name; protected by fil_system.mutex and log_sys.mutex */
+  /** file name; protected by fil_system.mutex and exclusive log_sys.latch */
   char *name;
   /** file handle */
   pfs_os_file_t handle;
@@ -1434,14 +1431,12 @@ public:
   /** nonzero if fil_node_open_file_low() should avoid moving the tablespace
   to the end of space_list, for FIFO policy of try_to_close() */
   ulint freeze_space_list;
+  /** list of all tablespaces */
   ilist<fil_space_t, space_list_tag_t> space_list;
-					/*!< list of all file spaces */
+  /** list of all tablespaces for which a FILE_MODIFY record has been written
+  since the latest redo log checkpoint.
+  Protected only by exclusive log_sys.latch. */
   ilist<fil_space_t, named_spaces_tag_t> named_spaces;
-					/*!< list of all file spaces
-					for which a FILE_MODIFY
-					record has been written since
-					the latest redo log checkpoint.
-					Protected only by log_sys.mutex. */
 
   /** list of all ENCRYPTED=DEFAULT tablespaces that need
   to be converted to the current value of innodb_encrypt_tables */

storage/innobase/include/log0log.h

@@ -37,6 +37,7 @@ Created 12/9/1995 Heikki Tuuri
 #include "os0file.h"
 #include "span.h"
 #include "my_atomic_wrapper.h"
+#include "srw_lock.h"
 #include <string>
 
 using st_::span;
@@ -57,16 +58,6 @@ static inline void delete_log_file(const char* suffix)
   os_file_delete_if_exists(innodb_log_file_key, path.c_str(), nullptr);
 }
 
-/** Calculate the recommended highest values for lsn - last_checkpoint_lsn
-and lsn - buf_pool.get_oldest_modification().
-@param[in]	file_size	requested innodb_log_file_size
-@retval true on success
-@retval false if the smallest log is too small to
-accommodate the number of OS threads in the database server */
-bool
-log_set_capacity(ulonglong file_size)
-	MY_ATTRIBUTE((warn_unused_result));
-
 struct completion_callback;
 
 /** Ensure that the log has been written to the log file up to a given
@@ -83,10 +74,10 @@ void log_write_up_to(lsn_t lsn, bool durable,
 void log_buffer_flush_to_disk(bool durable= true);
 
 
-/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */
+/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */
 ATTRIBUTE_COLD void log_write_and_flush_prepare();
 
-/** Durably write the log up to log_sys.lsn() and release log_sys.mutex. */
+/** Durably write the log up to log_sys.get_lsn(). */
 ATTRIBUTE_COLD void log_write_and_flush();
 
 /** Make a checkpoint */
@@ -202,35 +193,38 @@ private:
   preflush buffer pool pages, or initiate a log checkpoint.
   This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */
   std::atomic<bool> check_flush_or_checkpoint_;
+
 public:
-  /** mutex protecting the log */
+  /** rw-lock protecting buf */
-  MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) mysql_mutex_t mutex;
+  MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) srw_lock latch;
 private:
   /** Last written LSN */
   lsn_t write_lsn;
 public:
-  /** first free offset within the log buffer in use */
-  size_t buf_free;
-  /** recommended maximum size of buf, after which the buffer is flushed */
-  size_t max_buf_free;
-  /** mutex that ensures that inserts into buf_pool.flush_list are in
-  LSN order; allows mtr_t::commit() to release log_sys.mutex earlier */
-  MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) mysql_mutex_t flush_order_mutex;
   /** log record buffer, written to by mtr_t::commit() */
   byte *buf;
   /** buffer for writing data to ib_logfile0, or nullptr if is_pmem()
   In write_buf(), buf and flush_buf are swapped */
   byte *flush_buf;
-  /** number of write requests (to buf); protected by mutex */
-  ulint write_to_buf;
   /** number of std::swap(buf, flush_buf) and writes from buf to log;
-  protected by mutex */
+  protected by latch.wr_lock() */
   ulint write_to_log;
-  /** number of waits in append_prepare() */
-  ulint waits;
   /** innodb_log_buffer_size (size of buf and flush_buf, in bytes) */
   size_t buf_size;
 
+private:
+  /** spin lock protecting lsn, buf_free in append_prepare() */
+  MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) srw_mutex lsn_lock;
+public:
+  /** first free offset within buf use; protected by lsn_lock */
+  Atomic_relaxed<size_t> buf_free;
+  /** number of write requests (to buf); protected by exclusive lsn_lock */
+  ulint write_to_buf;
+  /** number of waits in append_prepare(); protected by lsn_lock */
+  ulint waits;
+  /** recommended maximum size of buf, after which the buffer is flushed */
+  size_t max_buf_free;
+
   /** log file size in bytes, including the header */
   lsn_t file_size;
 private:
@@ -272,11 +266,11 @@ public:
 					/*!< this is the maximum allowed value
 					for lsn - last_checkpoint_lsn when a
 					new query step is started */
-  /** latest completed checkpoint (protected by log_sys.mutex) */
+  /** latest completed checkpoint (protected by latch.wr_lock()) */
   Atomic_relaxed<lsn_t> last_checkpoint_lsn;
 	lsn_t		next_checkpoint_lsn;
 					/*!< next checkpoint lsn */
-  /** next checkpoint number (protected by mutex) */
+  /** next checkpoint number (protected by latch.wr_lock()) */
   ulint next_checkpoint_no;
   /** number of pending checkpoint writes */
   ulint n_pending_checkpoint_writes;
@@ -299,6 +293,9 @@ public:
 
   void close_file();
 
+  /** Calculate the checkpoint safety margins. */
+  static void set_capacity();
+
   lsn_t get_lsn(std::memory_order order= std::memory_order_relaxed) const
   { return lsn.load(order); }
   void set_lsn(lsn_t lsn) { this->lsn.store(lsn, std::memory_order_release); }
@@ -310,17 +307,17 @@ public:
   /** Initialize the LSN on initial log file creation. */
   lsn_t init_lsn() noexcept
   {
-    mysql_mutex_lock(&mutex);
+    latch.wr_lock(SRW_LOCK_CALL);
     const lsn_t lsn{get_lsn()};
     flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed);
     write_lsn= lsn;
-    mysql_mutex_unlock(&mutex);
+    latch.wr_unlock();
     return lsn;
   }
 
   void set_recovered_lsn(lsn_t lsn) noexcept
   {
-    mysql_mutex_assert_owner(&mutex);
+    ut_ad(latch.is_write_locked());
     write_lsn= lsn;
     this->lsn.store(lsn, std::memory_order_relaxed);
     flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed);
@@ -360,20 +357,29 @@ public:
   static size_t get_block_size() { return 512; }
 #endif
 
+private:
+  /** Wait in append_prepare() for buffer to become available
+  @param ex   whether log_sys.latch is exclusively locked */
+  ATTRIBUTE_COLD static void append_prepare_wait(bool ex) noexcept;
+public:
   /** Reserve space in the log buffer for appending data.
-  @param size   upper limit of the length of the data to append(), in bytes
+  @tparam pmem  log_sys.is_pmem()
-  @return the current LSN */
+  @param size   total length of the data to append(), in bytes
-  inline lsn_t append_prepare(size_t size) noexcept;
+  @param ex     whether log_sys.latch is exclusively locked
+  @return the start LSN and the buffer position for append() */
+  template<bool pmem>
+  inline std::pair<lsn_t,byte*> append_prepare(size_t size, bool ex) noexcept;
 
   /** Append a string of bytes to the redo log.
+  @param d     destination
   @param s     string of bytes
   @param size  length of str, in bytes */
-  void append(const void *s, size_t size) noexcept
+  void append(byte *&d, const void *s, size_t size) noexcept
   {
-    mysql_mutex_assert_owner(&mutex);
+    ut_ad(latch.is_locked());
-    ut_ad(buf_free + size <= (is_pmem() ? file_size : buf_size));
+    ut_ad(d + size <= buf + (is_pmem() ? file_size : buf_size));
-    memcpy(buf + buf_free, s, size);
+    memcpy(d, s, size);
-    buf_free+= size;
+    d+= size;
   }
 
   /** Set the log file format. */
@@ -409,14 +415,15 @@ public:
     return START_OFFSET + (lsn - first_lsn) % capacity();
   }
 
-  /** Write checkpoint information to the log header and release mutex.
+  /** Write checkpoint information and invoke latch.wr_unlock().
   @param end_lsn    start LSN of the FILE_CHECKPOINT mini-transaction */
   inline void write_checkpoint(lsn_t end_lsn) noexcept;
 
-  /** Write buf to ib_logfile0 and release mutex.
+  /** Write buf to ib_logfile0.
+  @tparam release_latch whether to invoke latch.wr_unlock()
   @return new write target
   @retval 0 if everything was written */
-  inline lsn_t write_buf() noexcept;
+  template<bool release_latch> inline lsn_t write_buf() noexcept;
 
   /** Create the log. */
   void create(lsn_t lsn) noexcept;

storage/innobase/include/log0recv.h

@@ -420,8 +420,8 @@ extern bool		recv_no_ibuf_operations;
 /** TRUE when recv_init_crash_recovery() has been called. */
 extern bool		recv_needed_recovery;
 #ifdef UNIV_DEBUG
-/** TRUE if writing to the redo log (mtr_commit) is forbidden.
+/** whether writing to the redo log is forbidden;
-Protected by log_sys.mutex. */
+protected by exclusive log_sys.latch. */
 extern bool recv_no_log_write;
 #endif /* UNIV_DEBUG */
 

storage/innobase/include/mtr0mtr.h

@@ -107,7 +107,7 @@ struct mtr_t {
   /** Commit a mini-transaction that did not modify any pages,
   but generated some redo log on a higher level, such as
   FILE_MODIFY records and an optional FILE_CHECKPOINT marker.
-  The caller must hold log_sys.mutex.
+  The caller must hold exclusive log_sys.latch.
   This is to be used at log_checkpoint().
   @param checkpoint_lsn   the log sequence number of a checkpoint, or 0
   @return current LSN */
@@ -624,10 +624,6 @@ private:
   @param type   extended record subtype; @see mrec_ext_t */
   inline void log_write_extended(const buf_block_t &block, byte type);
 
-  /** Prepare to write the mini-transaction log to the redo log buffer.
-  @return number of bytes to write in finish_write() */
-  inline size_t prepare_write();
-
   /** Write a FILE_MODIFY record when a non-predefined persistent
   tablespace was modified for the first time since fil_names_clear(). */
   ATTRIBUTE_NOINLINE ATTRIBUTE_COLD void name_write();
@@ -636,9 +632,15 @@ private:
   ATTRIBUTE_NOINLINE void encrypt();
 
   /** Append the redo log records to the redo log buffer.
-  @param len   number of bytes to write
+  @param ex   whether log_sys.latch is already exclusively locked
   @return {start_lsn,flush_ahead} */
-  std::pair<lsn_t,page_flush_ahead> finish_write(size_t len);
+  std::pair<lsn_t,page_flush_ahead> do_write(bool ex);
+
+  /** Append the redo log records to the redo log buffer.
+  @param len   number of bytes to write
+  @param ex    whether log_sys.latch is exclusively locked
+  @return {start_lsn,flush_ahead} */
+  std::pair<lsn_t,page_flush_ahead> finish_write(size_t len, bool ex);
 
   /** Release the resources */
   inline void release_resources();

storage/innobase/include/mtr0mtr.inl

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2021, MariaDB Corporation.
+Copyright (c) 2017, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -33,7 +33,8 @@ inline bool mtr_t::is_block_dirtied(const buf_block_t *block)
   ut_ad(block->page.in_file());
   ut_ad(block->page.frame);
   ut_ad(block->page.buf_fix_count());
-  return block->page.oldest_modification() <= 1;
+  return block->page.oldest_modification() <= 1 &&
+    block->page.id().space() < SRV_TMP_SPACE_ID;
 }
 
 /**
@@ -48,12 +49,11 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type)
 	ut_ad(ut_is_2pow(type));
 
 	/* If this mtr has x-fixed a clean page then we set
-	the made_dirty flag. This tells us if we need to
+	the made_dirty flag. This tells mtr_t::commit()
-	grab log_sys.flush_order_mutex at mtr_t::commit() so that we
+	to hold log_sys.latch longer. */
-	can insert the dirtied page into the flush list. */
 
-	if ((type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX)
+	if (!m_made_dirty
-	    && !m_made_dirty) {
+            && (type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX)) {
 
 		m_made_dirty = is_block_dirtied(
 			reinterpret_cast<const buf_block_t*>(object));

storage/innobase/include/srv0srv.h

@@ -86,9 +86,6 @@ struct srv_stats_t
 	/** Count the amount of data written in total (in bytes) */
 	ulint_ctr_1_t		data_written;
 
-	/** Store the number of write requests issued */
-	ulint_ctr_1_t		buf_pool_write_requests;
-
 	/** Number of buffer pool reads that led to the reading of
 	a disk page */
 	ulint_ctr_1_t		buf_pool_reads;
@@ -684,7 +681,6 @@ struct export_var_t{
 	ulint innodb_buffer_pool_pages_old;
 	ulint innodb_buffer_pool_read_requests;	/*!< buf_pool.stat.n_page_gets */
 	ulint innodb_buffer_pool_reads;		/*!< srv_buf_pool_reads */
-	ulint innodb_buffer_pool_write_requests;/*!< srv_stats.buf_pool_write_requests */
 	ulint innodb_buffer_pool_read_ahead_rnd;/*!< srv_read_ahead_rnd */
 	ulint innodb_buffer_pool_read_ahead;	/*!< srv_read_ahead */
 	ulint innodb_buffer_pool_read_ahead_evicted;/*!< srv_read_ahead evicted*/

storage/innobase/include/srw_lock.h

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2020, 2021, MariaDB Corporation.
+Copyright (c) 2020, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software

storage/innobase/include/univ.i

@@ -517,8 +517,6 @@ extern mysql_pfs_key_t fts_pll_tokenize_mutex_key;
 extern mysql_pfs_key_t ibuf_bitmap_mutex_key;
 extern mysql_pfs_key_t ibuf_mutex_key;
 extern mysql_pfs_key_t ibuf_pessimistic_insert_mutex_key;
-extern mysql_pfs_key_t log_sys_mutex_key;
-extern mysql_pfs_key_t log_flush_order_mutex_key;
 extern mysql_pfs_key_t recalc_pool_mutex_key;
 extern mysql_pfs_key_t purge_sys_pq_mutex_key;
 extern mysql_pfs_key_t recv_sys_mutex_key;
@@ -547,5 +545,6 @@ extern mysql_pfs_key_t index_tree_rw_lock_key;
 extern mysql_pfs_key_t index_online_log_key;
 extern mysql_pfs_key_t trx_sys_rw_lock_key;
 extern mysql_pfs_key_t lock_latch_key;
+extern mysql_pfs_key_t log_latch_key;
 # endif /* UNIV_PFS_RWLOCK */
 #endif /* HAVE_PSI_INTERFACE */

storage/innobase/log/log0log.cc

@@ -74,52 +74,24 @@ log_t	log_sys;
 #define LOG_BUF_FLUSH_MARGIN	((4 * 4096) /* cf. log_t::append_prepare() */ \
 				 + (4U << srv_page_size_shift))
 
-/** Calculate the recommended highest values for lsn - last_checkpoint_lsn
+void log_t::set_capacity()
-and lsn - buf_pool.get_oldest_modification().
-@param[in]	file_size	requested innodb_log_file_size
-@retval true on success
-@retval false if the smallest log group is too small to
-accommodate the number of OS threads in the database server */
-bool
-log_set_capacity(ulonglong file_size)
 {
-	mysql_mutex_assert_owner(&log_sys.mutex);
+	ut_ad(log_sys.latch.is_write_locked());
 
 	/* Margin for the free space in the smallest log, before a new query
 	step which modifies the database, is started */
-	const size_t LOG_CHECKPOINT_FREE_PER_THREAD = 4U
-						      << srv_page_size_shift;
-	const size_t LOG_CHECKPOINT_EXTRA_FREE = 8U << srv_page_size_shift;
 
-	lsn_t		margin;
+	lsn_t smallest_capacity = srv_log_file_size - log_t::START_OFFSET;
-	ulint		free;
-
-	lsn_t smallest_capacity = file_size - log_t::START_OFFSET;
 	/* Add extra safety */
 	smallest_capacity -= smallest_capacity / 10;
 
-	/* For each OS thread we must reserve so much free space in the
+	lsn_t margin = smallest_capacity - (48 << srv_page_size_shift);
-	smallest log group that it can accommodate the log entries produced
+	margin -= margin / 10;	/* Add still some extra safety */
-	by single query steps: running out of free log space is a serious
-	system error which requires rebooting the database. */
-
-	free = LOG_CHECKPOINT_FREE_PER_THREAD * 10
-		+ LOG_CHECKPOINT_EXTRA_FREE;
-	if (free >= smallest_capacity / 2) {
-		sql_print_error("InnoDB: innodb_log_file_size is too small."
-				" %s", INNODB_PARAMETERS_MSG);
-		return false;
-	}
-
-	margin = smallest_capacity - free;
-	margin = margin - margin / 10;	/* Add still some extra safety */
 
 	log_sys.log_capacity = smallest_capacity;
 
 	log_sys.max_modified_age_async = margin - margin / 8;
 	log_sys.max_checkpoint_age = margin;
-
-	return(true);
 }
 
 /** Initialize the redo log subsystem. */
@@ -128,14 +100,8 @@ void log_t::create()
   ut_ad(this == &log_sys);
   ut_ad(!is_initialised());
 
-#if defined(__aarch64__)
+  latch.SRW_LOCK_INIT(log_latch_key);
-  mysql_mutex_init(log_sys_mutex_key, &mutex, MY_MUTEX_INIT_FAST);
+  lsn_lock.init();
-  mysql_mutex_init(
-    log_flush_order_mutex_key, &flush_order_mutex, MY_MUTEX_INIT_FAST);
-#else
-  mysql_mutex_init(log_sys_mutex_key, &mutex, nullptr);
-  mysql_mutex_init(log_flush_order_mutex_key, &flush_order_mutex, nullptr);
-#endif
 
   /* LSN 0 and 1 are reserved; @see buf_page_t::oldest_modification_ */
   lsn.store(FIRST_LSN, std::memory_order_relaxed);
@@ -272,7 +238,7 @@ void log_t::attach(log_file_t file, os_offset_t size)
 
 void log_t::create(lsn_t lsn) noexcept
 {
-  mysql_mutex_assert_owner(&mutex);
+  ut_ad(latch.is_write_locked());
   ut_ad(!recv_no_log_write);
   ut_ad(is_latest());
   ut_ad(this == &log_sys);
@@ -516,7 +482,6 @@ static size_t log_pad(lsn_t lsn, size_t pad, byte *begin, byte *extra)
 inline void log_t::persist(lsn_t lsn) noexcept
 {
   ut_ad(is_pmem());
-  mysql_mutex_assert_not_owner(&mutex);
   ut_ad(!write_lock.is_owner());
   ut_ad(!flush_lock.is_owner());
 
@@ -551,13 +516,13 @@ inline void log_t::persist(lsn_t lsn) noexcept
 }
 #endif
 
-/** Write buf to ib_logfile0 and release mutex.
+/** Write buf to ib_logfile0.
+@tparam release_latch whether to invoke latch.wr_unlock()
 @return new write target
 @retval 0 if everything was written */
-inline lsn_t log_t::write_buf() noexcept
+template<bool release_latch> inline lsn_t log_t::write_buf() noexcept
 {
-  mysql_mutex_assert_owner(&mutex);
+  ut_ad(latch.is_write_locked());
-
   ut_ad(!srv_read_only_mode);
   ut_ad(!is_pmem());
 
@@ -565,7 +530,8 @@ inline lsn_t log_t::write_buf() noexcept
 
   if (write_lsn >= lsn)
   {
-    mysql_mutex_unlock(&mutex);
+    if (release_latch)
+      latch.wr_unlock();
     ut_ad(write_lsn == lsn);
   }
   else
@@ -581,31 +547,33 @@ inline lsn_t log_t::write_buf() noexcept
     const byte *write_buf{buf};
     size_t length{buf_free};
     ut_ad(length >= (calc_lsn_offset(write_lsn) & block_size_1));
-    buf_free&= block_size_1;
+    const size_t new_buf_free{length & block_size_1};
-    ut_ad(buf_free == ((lsn - first_lsn) & block_size_1));
+    buf_free= new_buf_free;
+    ut_ad(new_buf_free == ((lsn - first_lsn) & block_size_1));
 
-    if (buf_free)
+    if (new_buf_free)
     {
 #if 0 /* TODO: Pad the last log block with dummy records. */
-      buf_free= log_pad(lsn, get_block_size() - buf_free,
+      buf_free= log_pad(lsn, get_block_size() - new_buf_free,
-                        buf + buf_free, flush_buf);
+                        buf + new_buf_free, flush_buf);
       ... /* TODO: Update the LSN and adjust other code. */
 #else
       /* The rest of the block will be written as garbage.
       (We want to avoid memset() while holding mutex.)
       This block will be overwritten later, once records beyond
       the current LSN are generated. */
-      MEM_MAKE_DEFINED(buf + length, get_block_size() - buf_free);
+      MEM_MAKE_DEFINED(buf + length, get_block_size() - new_buf_free);
       buf[length]= 0; /* allow recovery to catch EOF faster */
       length&= ~block_size_1;
-      memcpy_aligned<16>(flush_buf, buf + length, (buf_free + 15) & ~15);
+      memcpy_aligned<16>(flush_buf, buf + length, (new_buf_free + 15) & ~15);
       length+= get_block_size();
 #endif
     }
 
     std::swap(buf, flush_buf);
     write_to_log++;
-    mysql_mutex_unlock(&mutex);
+    if (release_latch)
+      latch.wr_unlock();
 
     if (UNIV_UNLIKELY(srv_shutdown_state > SRV_SHUTDOWN_INITIATED))
     {
@@ -616,11 +584,6 @@ inline lsn_t log_t::write_buf() noexcept
     /* Do the write to the log file */
     log_write_buf(write_buf, length, offset);
     write_lsn= lsn;
-    if (srv_file_flush_method == SRV_O_DSYNC)
-    {
-      flushed_to_disk_lsn.store(lsn, std::memory_order_release);
-      log_flush_notify(lsn);
-    }
   }
 
   return write_lock.release(lsn);
@@ -630,7 +593,7 @@ inline bool log_t::flush(lsn_t lsn) noexcept
 {
   ut_ad(lsn >= get_flushed_lsn());
   flush_lock.set_pending(lsn);
-  const bool success{log.flush()};
+  const bool success{srv_file_flush_method == SRV_O_DSYNC || log.flush()};
   if (UNIV_LIKELY(success))
   {
     flushed_to_disk_lsn.store(lsn, std::memory_order_release);
@@ -646,10 +609,7 @@ inline bool log_t::flush(lsn_t lsn) noexcept
 static lsn_t log_flush(lsn_t lsn)
 {
   ut_ad(!log_sys.is_pmem());
-
-  if (srv_file_flush_method != SRV_O_DSYNC)
   ut_a(log_sys.flush(lsn));
-
   DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE(););
   return flush_lock.release(lsn);
 }
@@ -698,8 +658,8 @@ repeat:
   if (write_lock.acquire(lsn, durable ? nullptr : callback) ==
       group_commit_lock::ACQUIRED)
   {
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
-    write_lsn= log_sys.write_buf();
+    write_lsn= log_sys.write_buf<true>();
   }
   else
     write_lsn= 0;
@@ -726,11 +686,9 @@ void log_buffer_flush_to_disk(bool durable)
   log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), durable);
 }
 
-/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */
+/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */
 ATTRIBUTE_COLD void log_write_and_flush_prepare()
 {
-  mysql_mutex_assert_not_owner(&log_sys.mutex);
-
   if (log_sys.is_pmem())
     return;
 
@@ -740,23 +698,18 @@ ATTRIBUTE_COLD void log_write_and_flush_prepare()
          group_commit_lock::ACQUIRED);
 }
 
-/** Durably write the log and release log_sys.mutex */
+/** Durably write the log up to log_sys.get_lsn(). */
 ATTRIBUTE_COLD void log_write_and_flush()
 {
   ut_ad(!srv_read_only_mode);
   if (!log_sys.is_pmem())
   {
-    const lsn_t write_lsn{log_sys.write_buf()};
+    log_sys.write_buf<false>();
-    const lsn_t flush_lsn{log_flush(write_lock.value())};
+    log_flush(write_lock.value());
-    if (write_lsn || flush_lsn)
-      log_write_up_to(std::max(write_lsn, flush_lsn), true, &dummy_callback);
   }
 #ifdef HAVE_PMEM
   else
-  {
-    mysql_mutex_unlock(&log_sys.mutex);
     log_sys.persist(log_sys.get_lsn());
-  }
 #endif
 }
 
@@ -766,11 +719,7 @@ Tries to establish a big enough margin of free space in the log buffer, such
 that a new log entry can be catenated without an immediate need for a flush. */
 ATTRIBUTE_COLD static void log_flush_margin()
 {
-  mysql_mutex_lock(&log_sys.mutex);
+  if (log_sys.buf_free > log_sys.max_buf_free)
-  const bool flush{log_sys.buf_free > log_sys.max_buf_free};
-  mysql_mutex_unlock(&log_sys.mutex);
-
-  if (flush)
     log_buffer_flush_to_disk(false);
 }
 
@@ -783,26 +732,27 @@ ATTRIBUTE_COLD static void log_checkpoint_margin()
 {
   while (log_sys.check_flush_or_checkpoint())
   {
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.rd_lock(SRW_LOCK_CALL);
     ut_ad(!recv_no_log_write);
 
     if (!log_sys.check_flush_or_checkpoint())
     {
 func_exit:
-      mysql_mutex_unlock(&log_sys.mutex);
+      log_sys.latch.rd_unlock();
       return;
     }
 
     const lsn_t lsn= log_sys.get_lsn();
     const lsn_t checkpoint= log_sys.last_checkpoint_lsn;
     const lsn_t sync_lsn= checkpoint + log_sys.max_checkpoint_age;
+
     if (lsn <= sync_lsn)
     {
       log_sys.set_check_flush_or_checkpoint(false);
       goto func_exit;
     }
 
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.rd_unlock();
 
     /* We must wait to prevent the tail of the log overwriting the head. */
     buf_flush_wait_flushed(std::min(sync_lsn, checkpoint + (1U << 20)));
@@ -952,9 +902,9 @@ wait_suspend_loop:
 	}
 
 	if (log_sys.is_initialised()) {
-		mysql_mutex_lock(&log_sys.mutex);
+		log_sys.latch.rd_lock(SRW_LOCK_CALL);
 		const ulint	n_write	= log_sys.n_pending_checkpoint_writes;
-		mysql_mutex_unlock(&log_sys.mutex);
+		log_sys.latch.rd_unlock();
 
 		if (n_write) {
 			if (srv_print_verbose_log && count > 600) {
@@ -995,7 +945,7 @@ wait_suspend_loop:
 			? SIZE_OF_FILE_CHECKPOINT + 8
 			: SIZE_OF_FILE_CHECKPOINT;
 
-		mysql_mutex_lock(&log_sys.mutex);
+		log_sys.latch.rd_lock(SRW_LOCK_CALL);
 
 		lsn = log_sys.get_lsn();
 
@@ -1003,7 +953,7 @@ wait_suspend_loop:
 			&& lsn != log_sys.last_checkpoint_lsn + sizeof_cp;
 		ut_ad(lsn >= log_sys.last_checkpoint_lsn);
 
-		mysql_mutex_unlock(&log_sys.mutex);
+		log_sys.latch.rd_unlock();
 
 		if (lsn_changed) {
 			goto loop;
@@ -1049,7 +999,7 @@ log_print(
 	double	time_elapsed;
 	time_t	current_time;
 
-	mysql_mutex_lock(&log_sys.mutex);
+	log_sys.latch.rd_lock(SRW_LOCK_CALL);
 
 	const lsn_t lsn= log_sys.get_lsn();
 	mysql_mutex_lock(&buf_pool.flush_list_mutex);
@@ -1087,7 +1037,7 @@ log_print(
 	log_sys.n_log_ios_old = log_sys.n_log_ios;
 	log_sys.last_printout_time = current_time;
 
-	mysql_mutex_unlock(&log_sys.mutex);
+	log_sys.latch.rd_unlock();
 }
 
 /**********************************************************************//**
@@ -1120,8 +1070,8 @@ void log_t::close()
   ut_ad(!flush_buf);
 #endif
 
-  mysql_mutex_destroy(&mutex);
+  latch.destroy();
-  mysql_mutex_destroy(&flush_order_mutex);
+  lsn_lock.destroy();
 
   recv_sys.close();
 

storage/innobase/log/log0recv.cc

@@ -62,7 +62,7 @@ recv_sys_t	recv_sys;
 bool	recv_needed_recovery;
 #ifdef UNIV_DEBUG
 /** TRUE if writing to the redo log (mtr_commit) is forbidden.
-Protected by log_sys.mutex. */
+Protected by log_sys.latch. */
 bool	recv_no_log_write = false;
 #endif /* UNIV_DEBUG */
 
@@ -907,6 +907,7 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p,
       node->deferred= true;
       if (!space->acquire())
         goto fail;
+      fil_names_dirty(space);
       const bool is_compressed= fil_space_t::is_compressed(flags);
 #ifdef _WIN32
       const bool is_sparse= is_compressed;
@@ -2235,7 +2236,9 @@ template<typename source>
 inline recv_sys_t::parse_mtr_result recv_sys_t::parse(store_t store, source &l)
   noexcept
 {
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_write_locked() ||
+        srv_operation == SRV_OPERATION_BACKUP ||
+        srv_operation == SRV_OPERATION_BACKUP_NO_DEFER);
   mysql_mutex_assert_owner(&mutex);
   ut_ad(log_sys.next_checkpoint_lsn);
   ut_ad(log_sys.is_latest());
@@ -2970,17 +2973,23 @@ set_start_lsn:
 
 	if (start_lsn) {
 		ut_ad(end_lsn >= start_lsn);
+		ut_ad(!block->page.oldest_modification());
 		mach_write_to_8(FIL_PAGE_LSN + frame, end_lsn);
-		if (UNIV_LIKELY(frame == block->page.frame)) {
+		if (UNIV_LIKELY(!block->page.zip.data)) {
 			mach_write_to_8(srv_page_size
 					- FIL_PAGE_END_LSN_OLD_CHKSUM
 					+ frame, end_lsn);
 		} else {
 			buf_zip_decompress(block, false);
 		}
-
+		/* The following is adapted from
-		buf_block_modify_clock_inc(block);
+		buf_pool_t::insert_into_flush_list() */
-		buf_flush_note_modification(block, start_lsn, end_lsn);
+		mysql_mutex_lock(&buf_pool.flush_list_mutex);
+		buf_pool.stat.flush_list_bytes+= block->physical_size();
+		block->page.set_oldest_modification(start_lsn);
+		UT_LIST_ADD_FIRST(buf_pool.flush_list, &block->page);
+		buf_pool.page_cleaner_wakeup();
+		mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 	} else if (free_page && init) {
 		/* There have been no operations that modify the page.
 		Any buffered changes must not be merged. A subsequent
@@ -3271,9 +3280,6 @@ void recv_sys_t::apply(bool last_batch)
         srv_operation == SRV_OPERATION_RESTORE ||
         srv_operation == SRV_OPERATION_RESTORE_EXPORT);
 
-#ifdef SAFE_MUTEX
-  DBUG_ASSERT(!last_batch == mysql_mutex_is_owner(&log_sys.mutex));
-#endif /* SAFE_MUTEX */
   mysql_mutex_assert_owner(&mutex);
 
   timespec abstime;
@@ -3283,15 +3289,15 @@ void recv_sys_t::apply(bool last_batch)
     if (is_corrupt_log())
       return;
     if (last_batch)
-    {
-      mysql_mutex_assert_not_owner(&log_sys.mutex);
       my_cond_wait(&cond, &mutex.m_mutex);
-    }
     else
     {
-      mysql_mutex_unlock(&mutex);
+      ut_ad(log_sys.latch.is_write_locked());
+      log_sys.latch.wr_unlock();
       set_timespec_nsec(abstime, 500000000ULL); /* 0.5s */
-      my_cond_timedwait(&cond, &log_sys.mutex.m_mutex, &abstime);
+      my_cond_timedwait(&cond, &mutex.m_mutex, &abstime);
+      mysql_mutex_unlock(&mutex);
+      log_sys.latch.wr_lock(SRW_LOCK_CALL);
       mysql_mutex_lock(&mutex);
     }
   }
@@ -3398,7 +3404,6 @@ next_free_block:
       {
         if (last_batch)
         {
-          mysql_mutex_assert_not_owner(&log_sys.mutex);
           if (!empty)
             my_cond_wait(&cond, &mutex.m_mutex);
           else
@@ -3412,9 +3417,12 @@ next_free_block:
         }
         else
         {
-          mysql_mutex_unlock(&mutex);
+          ut_ad(log_sys.latch.is_write_locked());
+          log_sys.latch.wr_unlock();
           set_timespec_nsec(abstime, 500000000ULL); /* 0.5s */
-          my_cond_timedwait(&cond, &log_sys.mutex.m_mutex, &abstime);
+          my_cond_timedwait(&cond, &mutex.m_mutex, &abstime);
+          mysql_mutex_unlock(&mutex);
+          log_sys.latch.wr_lock(SRW_LOCK_CALL);
           mysql_mutex_lock(&mutex);
         }
         continue;
@@ -3432,10 +3440,9 @@ next_free_block:
   else
   {
     mlog_init.reset();
-    mysql_mutex_unlock(&log_sys.mutex);
+    log_sys.latch.wr_unlock();
   }
 
-  mysql_mutex_assert_not_owner(&log_sys.mutex);
   mysql_mutex_unlock(&mutex);
 
   if (last_batch && srv_operation != SRV_OPERATION_RESTORE &&
@@ -3451,7 +3458,7 @@ next_free_block:
   if (!last_batch)
   {
     buf_pool_invalidate();
-    mysql_mutex_lock(&log_sys.mutex);
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
   }
 #ifdef HAVE_PMEM
   else if (log_sys.is_pmem())
@@ -3511,7 +3518,7 @@ static bool recv_scan_log(bool last_phase)
 
   for (ut_d(lsn_t source_offset= 0);;)
   {
-    mysql_mutex_assert_owner(&log_sys.mutex);
+    ut_ad(log_sys.latch.is_write_locked());
 #ifdef UNIV_DEBUG
     const bool wrap{source_offset + recv_sys.len == log_sys.file_size};
 #endif
@@ -3868,7 +3875,7 @@ recv_init_crash_recovery_spaces(bool rescan, bool& missing_tablespace)
 static dberr_t recv_rename_files()
 {
   mysql_mutex_assert_owner(&recv_sys.mutex);
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_write_locked());
 
   dberr_t err= DB_SUCCESS;
 
@@ -3963,20 +3970,16 @@ dberr_t recv_recovery_from_checkpoint_start()
 
 	recv_sys.recovery_on = true;
 
-	mysql_mutex_lock(&log_sys.mutex);
+	log_sys.latch.wr_lock(SRW_LOCK_CALL);
 
 	dberr_t err = recv_sys.find_checkpoint();
         if (err != DB_SUCCESS) {
 early_exit:
-		mysql_mutex_unlock(&log_sys.mutex);
+		log_sys.latch.wr_unlock();
 		return err;
 	}
 
-	if (!log_set_capacity(srv_log_file_size)) {
+	log_sys.set_capacity();
-err_exit:
-		err = DB_ERROR;
-		goto early_exit;
-	}
 
 	/* Start reading the log from the checkpoint lsn. The variable
 	contiguous_lsn contains an lsn up to which the log is known to
@@ -4103,7 +4106,9 @@ read_only_recovery:
 	}
 
 	if (recv_sys.lsn < log_sys.next_checkpoint_lsn) {
-		goto err_exit;
+err_exit:
+		err = DB_ERROR;
+		goto early_exit;
 	}
 
 	if (!srv_read_only_mode && log_sys.is_latest()) {
@@ -4142,7 +4147,6 @@ read_only_recovery:
 		err = recv_rename_files();
 	}
 	mysql_mutex_unlock(&recv_sys.mutex);
-	mysql_mutex_unlock(&log_sys.mutex);
 
 	recv_lsn_checks_on = true;
 
@@ -4154,6 +4158,7 @@ read_only_recovery:
 		err = DB_CORRUPTION;
 	}
 
+	log_sys.latch.wr_unlock();
 	return err;
 }
 

storage/innobase/mtr/mtr0mtr.cc

@@ -312,16 +312,10 @@ struct DebugCheck {
 struct ReleaseBlocks
 {
   const lsn_t start, end;
-#ifdef UNIV_DEBUG
+  mutable size_t modified;
-  const mtr_buf_t &memo;
+  ReleaseBlocks(lsn_t start, lsn_t end) : start(start), end(end), modified(0)
-
-  ReleaseBlocks(lsn_t start, lsn_t end, const mtr_buf_t &memo) :
-    start(start), end(end), memo(memo)
-#else /* UNIV_DEBUG */
-  ReleaseBlocks(lsn_t start, lsn_t end, const mtr_buf_t&) :
-    start(start), end(end)
-#endif /* UNIV_DEBUG */
   {
+    ut_ad(!srv_read_only_mode);
     ut_ad(start);
     ut_ad(end);
   }
@@ -340,8 +334,25 @@ struct ReleaseBlocks
       return true;
     }
 
-    buf_flush_note_modification(static_cast<buf_block_t*>(slot->object),
+    modified++;
-                                start, end);
+    buf_block_t *b= static_cast<buf_block_t*>(slot->object);
+    ut_d(const auto s= b->page.state());
+    ut_ad(s > buf_page_t::FREED);
+    ut_ad(s < buf_page_t::READ_FIX);
+    ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= end);
+    mach_write_to_8(b->page.frame + FIL_PAGE_LSN, end);
+    if (UNIV_LIKELY_NULL(b->page.zip.data))
+      memcpy_aligned<8>(FIL_PAGE_LSN + b->page.zip.data,
+                        FIL_PAGE_LSN + b->page.frame, 8);
+
+    const lsn_t oldest_modification= b->page.oldest_modification();
+
+    if (oldest_modification > 1)
+      ut_ad(oldest_modification <= start);
+    else if (fsp_is_system_temporary(b->page.id().space()))
+      b->page.set_temp_modified();
+    else
+      buf_pool.insert_into_flush_list(b, start);
     return true;
   }
 };
@@ -401,18 +412,21 @@ void mtr_t::commit()
 
     std::pair<lsn_t,page_flush_ahead> lsns;
 
-    if (const auto len= prepare_write())
+    if (UNIV_LIKELY(m_log_mode == MTR_LOG_ALL))
-      lsns= finish_write(len);
+    {
+      lsns= do_write(false);
+      if (!m_made_dirty)
+        log_sys.latch.rd_unlock();
+    }
     else
+    {
+      ut_ad(m_log_mode == MTR_LOG_NO_REDO);
+      ut_ad(m_log.size() == 0);
+      m_commit_lsn= log_sys.get_lsn();
       lsns= { m_commit_lsn, PAGE_FLUSH_NO };
-
+      if (UNIV_UNLIKELY(m_made_dirty)) /* This should be IMPORT TABLESPACE */
-    if (m_made_dirty)
+        log_sys.latch.rd_lock(SRW_LOCK_CALL);
-      mysql_mutex_lock(&log_sys.flush_order_mutex);
+    }
-
-    /* It is now safe to release log_sys.mutex because the
-    buf_pool.flush_order_mutex will ensure that we are the first one
-    to insert into buf_pool.flush_list. */
-    mysql_mutex_unlock(&log_sys.mutex);
 
     if (m_freed_pages)
     {
@@ -436,16 +450,23 @@ void mtr_t::commit()
     else
       ut_ad(!m_freed_space);
 
-    m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>
+    ReleaseBlocks rb{lsns.first, m_commit_lsn};
-                                     (ReleaseBlocks(lsns.first, m_commit_lsn,
+    m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>(rb));
-                                                    m_memo)));
     if (m_made_dirty)
-      mysql_mutex_unlock(&log_sys.flush_order_mutex);
+      log_sys.latch.rd_unlock();
 
     m_memo.for_each_block_in_reverse(CIterate<ReleaseLatches>());
 
     if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO))
       buf_flush_ahead(m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC);
+
+    if (rb.modified)
+    {
+      mysql_mutex_lock(&buf_pool.flush_list_mutex);
+      buf_pool.flush_list_requests+= rb.modified;
+      buf_pool.page_cleaner_wakeup();
+      mysql_mutex_unlock(&buf_pool.flush_list_mutex);
+    }
   }
   else
     m_memo.for_each_block_in_reverse(CIterate<ReleaseAll>());
@@ -514,12 +535,13 @@ void mtr_t::commit_shrink(fil_space_t &space)
   ut_ad(UT_LIST_GET_LEN(space.chain) == 1);
 
   log_write_and_flush_prepare();
+  log_sys.latch.wr_lock(SRW_LOCK_CALL);
 
-  const lsn_t start_lsn= finish_write(prepare_write()).first;
+  const lsn_t start_lsn= do_write(true).first;
 
-  mysql_mutex_lock(&log_sys.flush_order_mutex);
   /* Durably write the reduced FSP_SIZE before truncating the data file. */
   log_write_and_flush();
+  ut_ad(log_sys.latch.is_write_locked());
 
   os_file_truncate(space.chain.start->name, space.chain.start->handle,
                    os_offset_t{space.size} << srv_page_size_shift, true);
@@ -548,9 +570,8 @@ void mtr_t::commit_shrink(fil_space_t &space)
   m_memo.for_each_block_in_reverse(CIterate<Shrink>{space});
 
   m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>
-                                   (ReleaseBlocks(start_lsn, m_commit_lsn,
+                                   (ReleaseBlocks{start_lsn, m_commit_lsn}));
-                                                  m_memo)));
+  log_sys.latch.wr_unlock();
-  mysql_mutex_unlock(&log_sys.flush_order_mutex);
 
   mysql_mutex_lock(&fil_system.mutex);
   ut_ad(space.is_being_truncated);
@@ -573,7 +594,7 @@ This is to be used at log_checkpoint().
 @return current LSN */
 lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn)
 {
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_write_locked());
   ut_ad(is_active());
   ut_ad(!is_inside_ibuf());
   ut_ad(m_log_mode == MTR_LOG_ALL);
@@ -607,7 +628,7 @@ lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn)
   m_crc= 0;
   m_log.for_each_block([this](const mtr_buf_t::block_t *b)
   { m_crc= my_crc32c(m_crc, b->begin(), b->used()); return true; });
-  finish_write(size);
+  finish_write(size, true);
   release_resources();
 
   if (checkpoint_lsn)
@@ -744,8 +765,6 @@ mtr_t::release_page(const void* ptr, mtr_memo_type_t type)
 	ut_ad(0);
 }
 
-static bool log_margin_warned;
-static time_t log_margin_warn_time;
 static bool log_close_warned;
 static time_t log_close_warn_time;
 
@@ -765,65 +784,73 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t age, lsn_t capacity)
   }
 }
 
-/** Reserve space in the log buffer for appending data.
+/** Wait in append_prepare() for buffer to become available
-@param size   upper limit of the length of the data to append(), in bytes
+@param ex   whether log_sys.latch is exclusively locked */
-@return the current LSN */
+ATTRIBUTE_COLD void log_t::append_prepare_wait(bool ex) noexcept
-inline lsn_t log_t::append_prepare(size_t size) noexcept
 {
-  mysql_mutex_assert_owner(&mutex);
+  log_sys.waits++;
+  log_sys.lsn_lock.wr_unlock();
 
-  lsn_t lsn= get_lsn();
+  if (ex)
+    log_sys.latch.wr_unlock();
+  else
+    log_sys.latch.rd_unlock();
 
-  if (UNIV_UNLIKELY(size > log_capacity))
+  DEBUG_SYNC_C("log_buf_size_exceeded");
+  log_buffer_flush_to_disk(log_sys.is_pmem());
+
+  if (ex)
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
+  else
+    log_sys.latch.rd_lock(SRW_LOCK_CALL);
+
+  log_sys.lsn_lock.wr_lock();
+}
+
+/** Reserve space in the log buffer for appending data.
+@tparam pmem  log_sys.is_pmem()
+@param size   total length of the data to append(), in bytes
+@param ex     whether log_sys.latch is exclusively locked
+@return the start LSN and the buffer position for append() */
+template<bool pmem>
+inline
+std::pair<lsn_t,byte*> log_t::append_prepare(size_t size, bool ex) noexcept
+{
+  ut_ad(latch.is_locked());
+  ut_ad(pmem == is_pmem());
+#ifndef _WIN32 // there is no accurate is_write_locked() on SRWLOCK
+  ut_ad(ex == latch.is_write_locked());
+#endif
+  const lsn_t checkpoint_margin{last_checkpoint_lsn + log_capacity - size};
+  const size_t avail{(pmem ? size_t(capacity()) : buf_size) - size};
+  lsn_lock.wr_lock(); /* Just use SRWLOCK or pthread_mutex_t */
+  write_to_buf++;
+
+  for (ut_d(int count= 50);
+       UNIV_UNLIKELY((pmem
+                      ? size_t(get_lsn() -
+                               get_flushed_lsn(std::memory_order_relaxed))
+                      : size_t{buf_free}) > avail); )
   {
-    time_t t= time(nullptr);
+    append_prepare_wait(ex);
-
+    ut_ad(count--);
-    /* return with warning output to avoid deadlock */
-    if (!log_margin_warned || difftime(t, log_margin_warn_time) > 15)
-    {
-      log_margin_warned= true;
-      log_margin_warn_time= t;
-
-      sql_print_error("InnoDB: innodb_log_file_size is too small "
-                      "for mini-transaction size %zu", size);
   }
-    goto throttle;
+
-  }
+  const lsn_t l{lsn.load(std::memory_order_relaxed)};
-  else if (UNIV_UNLIKELY(lsn + size > last_checkpoint_lsn + log_capacity))
+  lsn.store(l + size, std::memory_order_relaxed);
-  throttle:
+  const size_t b{buf_free};
+  size_t new_buf_free{b};
+  new_buf_free+= size;
+  if (pmem && new_buf_free >= file_size)
+    new_buf_free-= size_t(capacity());
+  buf_free= new_buf_free;
+  lsn_lock.wr_unlock();
+
+  if (UNIV_UNLIKELY(l > checkpoint_margin) ||
+      (!pmem && b >= max_buf_free))
     set_check_flush_or_checkpoint();
 
-  if (is_pmem())
+  return {l, &buf[b]};
-  {
-    for (ut_d(int count= 50); capacity() - size <
-           size_t(lsn - flushed_to_disk_lsn.load(std::memory_order_relaxed)); )
-    {
-      waits++;
-      mysql_mutex_unlock(&mutex);
-      DEBUG_SYNC_C("log_buf_size_exceeded");
-      log_write_up_to(lsn, true);
-      ut_ad(count--);
-      mysql_mutex_lock(&mutex);
-      lsn= get_lsn();
-    }
-    return lsn;
-  }
-
-  /* Calculate the amount of free space needed. */
-  size= (4 * 4096) - size + log_sys.buf_size;
-
-  for (ut_d(int count= 50); UNIV_UNLIKELY(buf_free > size); )
-  {
-    waits++;
-    mysql_mutex_unlock(&mutex);
-    DEBUG_SYNC_C("log_buf_size_exceeded");
-    log_write_up_to(lsn, false);
-    ut_ad(count--);
-    mysql_mutex_lock(&mutex);
-    lsn= get_lsn();
-  }
-
-  return lsn;
 }
 
 /** Finish appending data to the log.
@@ -831,9 +858,7 @@ inline lsn_t log_t::append_prepare(size_t size) noexcept
 @return whether buf_flush_ahead() will have to be invoked */
 static mtr_t::page_flush_ahead log_close(lsn_t lsn) noexcept
 {
-  mysql_mutex_assert_owner(&log_sys.mutex);
+  ut_ad(log_sys.latch.is_locked());
-  log_sys.write_to_buf++;
-  log_sys.set_lsn(lsn);
 
   const lsn_t checkpoint_age= lsn - log_sys.last_checkpoint_lsn;
 
@@ -850,17 +875,11 @@ static mtr_t::page_flush_ahead log_close(lsn_t lsn) noexcept
   return mtr_t::PAGE_FLUSH_SYNC;
 }
 
-inline size_t mtr_t::prepare_write()
+std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::do_write(bool ex)
 {
   ut_ad(!recv_no_log_write);
-  if (UNIV_UNLIKELY(m_log_mode != MTR_LOG_ALL))
+  ut_ad(m_log_mode == MTR_LOG_ALL);
-  {
+  ut_ad(!ex || log_sys.latch.is_write_locked());
-    ut_ad(m_log_mode == MTR_LOG_NO_REDO);
-    ut_ad(m_log.size() == 0);
-    mysql_mutex_lock(&log_sys.mutex);
-    m_commit_lsn= log_sys.get_lsn();
-    return 0;
-  }
 
   size_t len= m_log.size() + 5;
   ut_ad(len > 5);
@@ -877,86 +896,93 @@ inline size_t mtr_t::prepare_write()
     { m_crc= my_crc32c(m_crc, b->begin(), b->used()); return true; });
   }
 
-  mysql_mutex_lock(&log_sys.mutex);
+  if (!ex)
+    log_sys.latch.rd_lock(SRW_LOCK_CALL);
 
-  if (m_user_space && !is_predefined_tablespace(m_user_space->id) &&
+  if (UNIV_UNLIKELY(m_user_space && !m_user_space->max_lsn &&
-      !m_user_space->max_lsn)
+                    !is_predefined_tablespace(m_user_space->id)))
+  {
+    if (!ex)
+    {
+      log_sys.latch.rd_unlock();
+      log_sys.latch.wr_lock(SRW_LOCK_CALL);
+      if (UNIV_LIKELY(!m_user_space->max_lsn))
         name_write();
+      std::pair<lsn_t,mtr_t::page_flush_ahead> p{finish_write(len, true)};
+      log_sys.latch.wr_unlock();
+      log_sys.latch.rd_lock(SRW_LOCK_CALL);
+      return p;
+    }
+    else
+      name_write();
+  }
 
-  return len;
+  return finish_write(len, ex);
 }
 
 /** Write the mini-transaction log to the redo log buffer.
+@param len   number of bytes to write
+@param ex    whether log_sys.latch is exclusively locked
 @return {start_lsn,flush_ahead} */
-std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::finish_write(size_t len)
+std::pair<lsn_t,mtr_t::page_flush_ahead>
+mtr_t::finish_write(size_t len, bool ex)
 {
   ut_ad(!recv_no_log_write);
   ut_ad(m_log_mode == MTR_LOG_ALL);
 
-  const lsn_t start_lsn= log_sys.append_prepare(len);
   const size_t size{m_commit_lsn ? 5U + 8U : 5U};
+  std::pair<lsn_t, byte*> start;
 
   if (!log_sys.is_pmem())
   {
-    m_log.for_each_block([](const mtr_buf_t::block_t *b)
+    start= log_sys.append_prepare<false>(len, ex);
-    { log_sys.append(b->begin(), b->used()); return true; });
+    m_log.for_each_block([&start](const mtr_buf_t::block_t *b)
-
+    { log_sys.append(start.second, b->begin(), b->used()); return true; });
-    if (log_sys.buf_free >= log_sys.max_buf_free)
-      log_sys.set_check_flush_or_checkpoint();
 
 #ifdef HAVE_PMEM
   write_trailer:
 #endif
-    log_sys.buf[log_sys.buf_free]=
+    *start.second++= log_sys.get_sequence_bit(start.first + len - size);
-      log_sys.get_sequence_bit(start_lsn + len - size);
     if (m_commit_lsn)
     {
-      byte *nonce= log_sys.buf + log_sys.buf_free + 1;
+      mach_write_to_8(start.second, m_commit_lsn);
-      mach_write_to_8(nonce, m_commit_lsn);
+      m_crc= my_crc32c(m_crc, start.second, 8);
-      m_crc= my_crc32c(m_crc, nonce, 8);
+      start.second+= 8;
-      mach_write_to_4(&log_sys.buf[log_sys.buf_free + 9], m_crc);
-      log_sys.buf_free+= 8 + 5;
-    }
-    else
-    {
-      mach_write_to_4(&log_sys.buf[log_sys.buf_free + 1], m_crc);
-      log_sys.buf_free+= 5;
     }
+    mach_write_to_4(start.second, m_crc);
   }
 #ifdef HAVE_PMEM
-  else if (UNIV_LIKELY(log_sys.buf_free + len < log_sys.file_size))
+  else
   {
-    m_log.for_each_block([](const mtr_buf_t::block_t *b)
+    start= log_sys.append_prepare<true>(len, ex);
-    { log_sys.append(b->begin(), b->used()); return true; });
+    if (UNIV_LIKELY(start.second + len <= &log_sys.buf[log_sys.file_size]))
+    {
+      m_log.for_each_block([&start](const mtr_buf_t::block_t *b)
+      { log_sys.append(start.second, b->begin(), b->used()); return true; });
       goto write_trailer;
     }
-  else
+    m_log.for_each_block([&start](const mtr_buf_t::block_t *b)
-  {
-    m_log.for_each_block([](const mtr_buf_t::block_t *b)
     {
       size_t size{b->used()};
-      const size_t size_left{log_sys.file_size - log_sys.buf_free};
+      const size_t size_left(&log_sys.buf[log_sys.file_size] - start.second);
       const byte *src= b->begin();
-      if (size <= size_left)
+      if (size > size_left)
-      {
-        ::memcpy(log_sys.buf + log_sys.buf_free, src, size);
-        log_sys.buf_free+= size;
-      }
-      else
       {
+        ::memcpy(start.second, src, size_left);
+        start.second= &log_sys.buf[log_sys.START_OFFSET];
+        src+= size_left;
         size-= size_left;
-        ::memcpy(log_sys.buf + log_sys.buf_free, src, size_left);
-        ::memcpy(log_sys.buf + log_sys.START_OFFSET, src + size_left, size);
-        log_sys.buf_free= log_sys.START_OFFSET + size;
       }
+      ::memcpy(start.second, src, size);
+      start.second+= size;
       return true;
     });
-    const size_t size_left{log_sys.file_size - log_sys.buf_free};
+    const size_t size_left(&log_sys.buf[log_sys.file_size] - start.second);
     if (size_left > size)
       goto write_trailer;
 
     byte tail[5 + 8];
-    tail[0]= log_sys.get_sequence_bit(start_lsn + len - size);
+    tail[0]= log_sys.get_sequence_bit(start.first + len - size);
 
     if (m_commit_lsn)
     {
@@ -967,15 +993,14 @@ std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::finish_write(size_t len)
     else
       mach_write_to_4(tail + 1, m_crc);
 
-    ::memcpy(log_sys.buf + log_sys.buf_free, tail, size_left);
+    ::memcpy(start.second, tail, size_left);
     ::memcpy(log_sys.buf + log_sys.START_OFFSET, tail + size_left,
              size - size_left);
-    log_sys.buf_free= log_sys.START_OFFSET + (size - size_left);
   }
 #endif
 
-  m_commit_lsn= start_lsn + len;
+  m_commit_lsn= start.first + len;
-  return {start_lsn, log_close(m_commit_lsn)};
+  return {start.first, log_close(m_commit_lsn)};
 }
 
 /** Find out whether a block was not X-latched by the mini-transaction */

storage/innobase/os/os0file.cc

@@ -1303,8 +1303,9 @@ use_o_direct:
 		default:
 			break;
 		}
+	}
 # ifdef __linux__
-	} else if (type == OS_LOG_FILE && !log_sys.is_opened()) {
+	else if (type == OS_LOG_FILE && !log_sys.is_opened()) {
 		struct stat st;
 		char b[20 + sizeof "/sys/dev/block/" ":"
 		       "/../queue/physical_block_size"];

storage/innobase/row/row0sel.cc

@@ -357,11 +357,16 @@ row_sel_sec_rec_is_for_clust_rec(
 			}
 
 			len = clust_len;
+			ulint prefix_len = ifield->prefix_len;
 			if (rec_offs_nth_extern(clust_offs, clust_pos)) {
+				/* BLOB can contain prefix. */
 				len -= BTR_EXTERN_FIELD_REF_SIZE;
+				if (!len) {
+					goto compare_blobs;
+				}
 			}
 
-			if (ulint prefix_len = ifield->prefix_len) {
+			if (prefix_len) {
 				len = dtype_get_at_most_n_mbchars(
 					col->prtype, col->mbminlen,
 					col->mbmaxlen, prefix_len, len,
@@ -374,6 +379,7 @@ row_sel_sec_rec_is_for_clust_rec(
 check_for_blob:
 				if (rec_offs_nth_extern(clust_offs,
 							clust_pos)) {
+compare_blobs:
 					if (!row_sel_sec_rec_is_for_blob(
 						    col->mtype, col->prtype,
 						    col->mbminlen,

storage/innobase/srv/srv0mon.cc

@@ -1371,6 +1371,7 @@ corresponding monitors are turned on/off/reset, and do appropriate
 mathematics to deduct the actual value. Please also refer to
 srv_export_innodb_status() for related global counters used by
 the existing status variables.*/
+TPOOL_SUPPRESS_TSAN
 void
 srv_mon_process_existing_counter(
 /*=============================*/
@@ -1405,7 +1406,7 @@ srv_mon_process_existing_counter(
 	/* innodb_buffer_pool_write_requests, the number of
 	write request */
 	case MONITOR_OVLD_BUF_POOL_WRITE_REQUEST:
-		value = srv_stats.buf_pool_write_requests;
+		value = buf_pool.flush_list_requests;
 		break;
 
 	/* innodb_buffer_pool_wait_free */
@@ -1714,10 +1715,10 @@ srv_mon_process_existing_counter(
 		break;
 
 	case MONITOR_LSN_CHECKPOINT_AGE:
-		mysql_mutex_lock(&log_sys.mutex);
+		log_sys.latch.rd_lock(SRW_LOCK_CALL);
 		value = static_cast<mon_type_t>(log_sys.get_lsn()
 						- log_sys.last_checkpoint_lsn);
-		mysql_mutex_unlock(&log_sys.mutex);
+		log_sys.latch.rd_unlock();
 		break;
 
 	case MONITOR_OVLD_BUF_OLDEST_LSN:

storage/innobase/srv/srv0srv.cc

@@ -1024,9 +1024,6 @@ srv_export_innodb_status(void)
 	export_vars.innodb_buffer_pool_read_requests
 		= buf_pool.stat.n_page_gets;
 
-	export_vars.innodb_buffer_pool_write_requests =
-		srv_stats.buf_pool_write_requests;
-
 	export_vars.innodb_buffer_pool_reads = srv_stats.buf_pool_reads;
 
 	export_vars.innodb_buffer_pool_read_ahead_rnd =
@@ -1167,13 +1164,13 @@ srv_export_innodb_status(void)
 
 	mysql_mutex_unlock(&srv_innodb_monitor_mutex);
 
-	mysql_mutex_lock(&log_sys.mutex);
+	log_sys.latch.rd_lock(SRW_LOCK_CALL);
 	export_vars.innodb_lsn_current = log_sys.get_lsn();
 	export_vars.innodb_lsn_flushed = log_sys.get_flushed_lsn();
 	export_vars.innodb_lsn_last_checkpoint = log_sys.last_checkpoint_lsn;
 	export_vars.innodb_checkpoint_max_age = static_cast<ulint>(
 		log_sys.max_checkpoint_age);
-	mysql_mutex_unlock(&log_sys.mutex);
+	log_sys.latch.rd_unlock();
 	export_vars.innodb_os_log_written = export_vars.innodb_lsn_current
 		- recv_sys.lsn;
 
@@ -1818,10 +1815,10 @@ void purge_coordinator_state::refresh(bool full)
     lsn_hwm= adaptive_purge_threshold + series[n_threads];
   }
 
-  mysql_mutex_lock(&log_sys.mutex);
+  log_sys.latch.rd_lock(SRW_LOCK_CALL);
   const lsn_t last= log_sys.last_checkpoint_lsn,
     max_age= log_sys.max_checkpoint_age;
-  mysql_mutex_unlock(&log_sys.mutex);
+  log_sys.latch.rd_unlock();
 
   lsn_age_factor= ulint(((log_sys.get_lsn() - last) * 100) / max_age);
 }

storage/innobase/srv/srv0start.cc

@@ -195,12 +195,8 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn,
 
 	DBUG_ASSERT(!buf_pool.any_io_pending());
 
-	mysql_mutex_lock(&log_sys.mutex);
+	log_sys.latch.wr_lock(SRW_LOCK_CALL);
-	if (!log_set_capacity(srv_log_file_size)) {
+	log_sys.set_capacity();
-err_exit:
-		mysql_mutex_unlock(&log_sys.mutex);
-		return DB_ERROR;
-	}
 
 	logfile0 = get_log_file_path(LOG_FILE_NAME_PREFIX)
 		.append(INIT_LOG_FILE0);
@@ -213,7 +209,9 @@ err_exit:
 
 	if (!ret) {
 		sql_print_error("InnoDB: Cannot create %s", logfile0.c_str());
-		goto err_exit;
+err_exit:
+		log_sys.latch.wr_unlock();
+		return DB_ERROR;
 	}
 
 	ret = os_file_set_size(logfile0.c_str(), file, srv_log_file_size);
@@ -244,7 +242,7 @@ err_exit:
 
 	/* Enable checkpoints in buf_flush_page_cleaner(). */
 	recv_sys.recovery_on = false;
-	mysql_mutex_unlock(&log_sys.mutex);
+	log_sys.latch.wr_unlock();
 
 	log_make_checkpoint();
 	log_buffer_flush_to_disk();
@@ -801,7 +799,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file()
   DBUG_EXECUTE_IF("innodb_log_abort_1", DBUG_RETURN(0););
   DBUG_PRINT("ib_log", ("After innodb_log_abort_1"));
 
-  mysql_mutex_lock(&log_sys.mutex);
+  log_sys.latch.wr_lock(SRW_LOCK_CALL);
   const bool latest_format{log_sys.is_latest()};
   lsn_t flushed_lsn{log_sys.get_lsn()};
 
@@ -846,7 +844,7 @@ same_size:
     }
   }
 
-  mysql_mutex_unlock(&log_sys.mutex);
+  log_sys.latch.wr_unlock();
 
   log_write_up_to(flushed_lsn, false);
 

storage/innobase/sync/srw_lock.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2020, 2021, MariaDB Corporation.
+Copyright (c) 2020, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software

storage/maria/ha_maria.cc

@@ -2798,7 +2798,7 @@ int ha_maria::delete_table(const char *name)
 
 void ha_maria::drop_table(const char *name)
 {
-  DBUG_ASSERT(file->s->temporary);
+  DBUG_ASSERT(!file || file->s->temporary);
   (void) ha_close();
   (void) maria_delete_table_files(name, 1, MY_WME);
 }

storage/maria/ma_create.c

@@ -719,9 +719,10 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     share.base.extra_options|= MA_EXTRA_OPTIONS_INSERT_ORDER;
   }
 
+  share.state.state.key_file_length= MY_ALIGN(info_length, maria_block_size);
   DBUG_PRINT("info", ("info_length: %u", info_length));
   /* There are only 16 bits for the total header length. */
-  if (info_length > 65535)
+  if (share.state.state.key_file_length > 65535)
   {
     my_printf_error(HA_WRONG_CREATE_OPTION,
                     "Aria table '%s' has too many columns and/or "
@@ -778,8 +779,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
 
   maria_set_all_keys_active(share.state.key_map, keys);
 
-  share.base.keystart = share.state.state.key_file_length=
+  share.base.keystart = share.state.state.key_file_length;
-    MY_ALIGN(info_length, maria_block_size);
   share.base.max_key_block_length= maria_block_size;
   share.base.max_key_length=ALIGN_SIZE(max_key_length+4);
   share.base.records=ci->max_rows;

storage/perfschema/my_thread.h

@@ -14,6 +14,10 @@
 #include <pthread_np.h>
 #endif
 
+#if defined(HAVE_INTEGER_PTHREAD_SELF)
+#include <cstdint>
+#endif
+
 typedef pthread_key_t thread_local_key_t;
 typedef pthread_t my_thread_handle;
 typedef pthread_attr_t my_thread_attr_t;

tpool/aio_simulated.cc

@@ -72,7 +72,7 @@ struct WinIoInit
 static WinIoInit win_io_init;
 
 
-int pread(const native_file_handle &h, void *buf, size_t count,
+SSIZE_T pread(const native_file_handle &h, void *buf, size_t count,
                  unsigned long long offset)
 {
   OVERLAPPED ov{};
@@ -81,6 +81,8 @@ int pread(const native_file_handle &h, void *buf, size_t count,
   ov.Offset= uli.LowPart;
   ov.OffsetHigh= uli.HighPart;
   ov.hEvent= win_get_syncio_event();
+  if (count > 0xFFFFFFFF)
+    count= 0xFFFFFFFF;
 
   if (ReadFile(h, buf, (DWORD) count, 0, &ov) ||
       (GetLastError() == ERROR_IO_PENDING))
@@ -93,7 +95,7 @@ int pread(const native_file_handle &h, void *buf, size_t count,
   return -1;
 }
 
-int pwrite(const native_file_handle &h, void *buf, size_t count,
+SSIZE_T pwrite(const native_file_handle &h, void *buf, size_t count,
                   unsigned long long offset)
 {
   OVERLAPPED ov{};
@@ -102,7 +104,8 @@ int pwrite(const native_file_handle &h, void *buf, size_t count,
   ov.Offset= uli.LowPart;
   ov.OffsetHigh= uli.HighPart;
   ov.hEvent= win_get_syncio_event();
-
+  if (count > 0xFFFFFFFF)
+    count= 0xFFFFFFFF;
   if (WriteFile(h, buf, (DWORD) count, 0, &ov) ||
       (GetLastError() == ERROR_IO_PENDING))
   {

tpool/tpool.h

@@ -245,9 +245,9 @@ create_thread_pool_win(int min_threads= DEFAULT_MIN_POOL_THREADS,
   opened with FILE_FLAG_OVERLAPPED, and bound to completion
   port.
 */
-int pwrite(const native_file_handle &h, void *buf, size_t count,
+SSIZE_T pwrite(const native_file_handle &h, void *buf, size_t count,
            unsigned long long offset);
-int pread(const native_file_handle &h, void *buf, size_t count,
+SSIZE_T pread(const native_file_handle &h, void *buf, size_t count,
           unsigned long long offset);
 HANDLE win_get_syncio_event();
 #endif