Merge 10.6 into 10.7

include/my_valgrind.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2010, 2020, MariaDB Corporation.
+/* Copyright (C) 2010, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -37,6 +37,11 @@
 # define MEM_GET_VBITS(a,b,len) __msan_copy_shadow(b,a,len)
 # define MEM_SET_VBITS(a,b,len) __msan_copy_shadow(a,b,len)
 # define REDZONE_SIZE 8
+# ifdef __linux__
+#  define MSAN_STAT_WORKAROUND(st) MEM_MAKE_DEFINED(st, sizeof(*st))
+# else
+#  define MSAN_STAT_WORKAROUND(st) ((void) 0)
+# endif
 #elif defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind)
 # include <valgrind/memcheck.h>
 # define HAVE_MEM_CHECK
@@ -49,6 +54,7 @@
 # define MEM_GET_VBITS(a,b,len) VALGRIND_GET_VBITS(a,b,len)
 # define MEM_SET_VBITS(a,b,len) VALGRIND_SET_VBITS(a,b,len)
 # define REDZONE_SIZE 8
+# define MSAN_STAT_WORKAROUND(st) ((void) 0)
 #elif defined(__SANITIZE_ADDRESS__) && (!defined(_MSC_VER) || defined (__clang__))
 # include <sanitizer/asan_interface.h>
 /* How to do manual poisoning:
@@ -62,6 +68,7 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define MEM_GET_VBITS(a,b,len) ((void) 0)
 # define MEM_SET_VBITS(a,b,len) ((void) 0)
+# define MSAN_STAT_WORKAROUND(st) ((void) 0)
 # define REDZONE_SIZE 8
 #else
 # define MEM_UNDEFINED(a,len) ((void) 0)
@@ -73,6 +80,7 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */
 # define MEM_GET_VBITS(a,b,len) ((void) 0)
 # define MEM_SET_VBITS(a,b,len) ((void) 0)
 # define REDZONE_SIZE 0
+# define MSAN_STAT_WORKAROUND(st) ((void) 0)
 #endif /* __has_feature(memory_sanitizer) */
 
 #ifdef HAVE_valgrind

mysys/my_lib.c

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2008, 2020, MariaDB Corporation.
+   Copyright (c) 2008, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -332,6 +332,13 @@ int my_fstat(File Filedes, MY_STAT *stat_area,
   DBUG_PRINT("my",("fd: %d  MyFlags: %lu", Filedes, MyFlags));
 #ifdef _WIN32
   DBUG_RETURN(my_win_fstat(Filedes, stat_area));
+#elif defined HAVE_valgrind
+  {
+    int s= fstat(Filedes, stat_area);
+    if (!s)
+      MSAN_STAT_WORKAROUND(stat_area);
+    DBUG_RETURN(s);
+  }
 #else
   DBUG_RETURN(fstat(Filedes, (struct stat *) stat_area));
 #endif
@@ -350,11 +357,14 @@ MY_STAT *my_stat(const char *path, MY_STAT *stat_area, myf my_flags)
                                            my_flags)))
       goto error;
 #ifndef _WIN32
-    if (! stat((char *) path, (struct stat *) stat_area) )
+  if (!stat((char *) path, (struct stat *) stat_area))
-      DBUG_RETURN(stat_area);
+  {
+    MSAN_STAT_WORKAROUND(stat_area);
+    DBUG_RETURN(stat_area);
+  }
 #else
-    if (! my_win_stat(path, stat_area) )
+  if (!my_win_stat(path, stat_area))
-      DBUG_RETURN(stat_area);
+    DBUG_RETURN(stat_area);
 #endif
   DBUG_PRINT("error",("Got errno: %d from stat", errno));
   my_errno= errno;

mysys/my_symlink.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2001, 2011, Oracle and/or its affiliates
-   Copyright (c) 2010, 2017, MariaDB
+   Copyright (c) 2010, 2022, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -113,7 +113,10 @@ int my_is_symlink(const char *filename __attribute__((unused)))
 {
 #if defined (HAVE_LSTAT) && defined (S_ISLNK)
   struct stat stat_buff;
-  return !lstat(filename, &stat_buff) && S_ISLNK(stat_buff.st_mode);
+  if (lstat(filename, &stat_buff))
+    return 0;
+  MSAN_STAT_WORKAROUND(&stat_buff);
+  return !!S_ISLNK(stat_buff.st_mode);
 #elif defined (_WIN32)
   DWORD dwAttr = GetFileAttributes(filename);
   return (dwAttr != INVALID_FILE_ATTRIBUTES) &&

sql/datadict.cc

@@ -163,6 +163,8 @@ cont:
     if (mysql_file_fstat(file, &state, MYF(MY_WME)))
       goto err;
 
+    MSAN_STAT_WORKAROUND(&state);
+
     if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME)))
       goto err;
 

sql/ddl_log.cc

@@ -2491,11 +2491,11 @@ bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry,
 
   if (unlikely(write_ddl_log_file_entry((*active_entry)->entry_pos)))
   {
+    sql_print_error("DDL_LOG: Failed to write entry %u",
+                    (*active_entry)->entry_pos);
     ddl_log_release_memory_entry(*active_entry);
     *active_entry= 0;
     error= TRUE;
-    sql_print_error("DDL_LOG: Failed to write entry %u",
-                    (*active_entry)->entry_pos);
   }
   DBUG_RETURN(error);
 }
@@ -2558,13 +2558,13 @@ bool ddl_log_write_execute_entry(uint first_entry,
              (*active_entry)->entry_pos, first_entry));
   if (write_ddl_log_file_entry((*active_entry)->entry_pos))
   {
+    sql_print_error("DDL_LOG: Error writing execute entry %u",
+                    (*active_entry)->entry_pos);
     if (got_free_entry)
     {
       ddl_log_release_memory_entry(*active_entry);
       *active_entry= 0;
     }
-    sql_print_error("DDL_LOG: Error writing execute entry %u",
-                    (*active_entry)->entry_pos);
     DBUG_RETURN(TRUE);
   }
   (void) ddl_log_sync_no_lock();

sql/discover.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2009, 2020, MariaDB Corporation.
+   Copyright (c) 2009, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -72,6 +72,7 @@ int readfrm(const char *name, const uchar **frmdata, size_t *len)
   error= 2;
   if (mysql_file_fstat(file, &state, MYF(0)))
     goto err;
+  MSAN_STAT_WORKAROUND(&state);
   read_len= (size_t)MY_MIN(FRM_MAX_SIZE, state.st_size); // safety
 
   // Read whole frm file

sql/parse_file.cc

@@ -464,6 +464,8 @@ sql_parse_prepare(const LEX_CSTRING *file_name, MEM_ROOT *mem_root,
     DBUG_RETURN(0);
   }
 
+  MSAN_STAT_WORKAROUND(&stat_info);
+
   if (stat_info.st_size > INT_MAX-1)
   {
     my_error(ER_FPARSER_TOO_BIG_FILE, MYF(0), file_name->str);

sql/sql_parse.cc

@@ -1188,7 +1188,7 @@ static enum enum_server_command fetch_command(THD *thd, char *packet)
     DISPATCH_COMMAND_CLOSE_CONNECTION  request of THD shutdown
           (s. dispatch_command() description)
   @retval
-    DISPATCH_COMMAND_WOULDBLOCK - need to wait for asyncronous operations
+    DISPATCH_COMMAND_WOULDBLOCK - need to wait for asynchronous operations
                                   to finish. Only returned if parameter
                                   'blocking' is false.
 */

storage/innobase/buf/buf0rea.cc

@@ -290,7 +290,7 @@ nothing_read:
 		/* Trx sys header is so low in the latching order that we play
 		safe and do not leave the i/o-completion to an asynchronous
 		i/o-thread. Change buffer pages must always be read with
-		syncronous i/o, to make sure they do not get involved in
+		synchronous i/o, to make sure they do not get involved in
 		thread deadlocks. */
 		sync = true;
 	}

storage/innobase/os/os0file.cc

@@ -942,7 +942,7 @@ os_file_status_posix(
 
 	if (!ret) {
 		/* file exists, everything OK */
-
+		MSAN_STAT_WORKAROUND(&statinfo);
 	} else if (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG) {
 		/* file does not exist */
 		return(true);
@@ -1531,8 +1531,10 @@ bool os_file_close_func(os_file_t file)
 os_offset_t
 os_file_get_size(os_file_t file)
 {
-	struct stat statbuf;
+  struct stat statbuf;
-	return fstat(file, &statbuf) ? os_offset_t(-1) : statbuf.st_size;
+  if (fstat(file, &statbuf)) return os_offset_t(-1);
+  MSAN_STAT_WORKAROUND(&statbuf);
+  return statbuf.st_size;
 }
 
 /** Gets a file size.
@@ -1549,6 +1551,7 @@ os_file_get_size(
 	int	ret = stat(filename, &s);
 
 	if (ret == 0) {
+		MSAN_STAT_WORKAROUND(&s);
 		file_size.m_total_size = s.st_size;
 		/* st_blocks is in 512 byte sized blocks */
 		file_size.m_alloc_size = s.st_blocks * 512;
@@ -1593,6 +1596,8 @@ os_file_get_status_posix(
 		return(DB_FAIL);
 	}
 
+	MSAN_STAT_WORKAROUND(statinfo);
+
 	switch (statinfo->st_mode & S_IFMT) {
 	case S_IFDIR:
 		stat_info->type = OS_FILE_TYPE_DIR;
@@ -2785,7 +2790,7 @@ os_file_set_eof(
 
 #endif /* !_WIN32*/
 
-/** Does a syncronous read or write depending upon the type specified
+/** Does a synchronous read or write depending upon the type specified
 In case of partial reads/writes the function tries
 NUM_RETRIES_ON_PARTIAL_IO times to read/write the complete data.
 @param[in]	type,		IO flags
@@ -3275,6 +3280,7 @@ fallback:
 		if (fstat(file, &statbuf)) {
 			err = errno;
 		} else {
+			MSAN_STAT_WORKAROUND(&statbuf);
 			os_offset_t current_size = statbuf.st_size;
 			if (current_size >= size) {
 				return true;
@@ -4158,7 +4164,10 @@ void fil_node_t::find_metadata(os_file_t file
 #else
   struct stat sbuf;
   if (!statbuf && !fstat(file, &sbuf))
+  {
+    MSAN_STAT_WORKAROUND(&sbuf);
     statbuf= &sbuf;
+  }
   if (statbuf)
     block_size= statbuf->st_blksize;
 # ifdef UNIV_LINUX
@@ -4191,6 +4200,7 @@ bool fil_node_t::read_page0()
   struct stat statbuf;
   if (fstat(handle, &statbuf))
     return false;
+  MSAN_STAT_WORKAROUND(&statbuf);
   os_offset_t size_bytes= statbuf.st_size;
 #else
   os_offset_t size_bytes= os_file_get_size(handle);

storage/innobase/trx/trx0purge.cc

@@ -119,7 +119,7 @@ TRANSACTIONAL_INLINE inline bool TrxUndoRsegsIterator::set_next()
 	trx_id_t last_trx_no, tail_trx_no;
 	{
 #ifdef SUX_LOCK_GENERIC
-		purge_sys.rseg->latch.rd_lock();
+		purge_sys.rseg->latch.rd_lock(SRW_LOCK_CALL);
 #else
 		transactional_shared_lock_guard<srw_spin_lock> rg
 			{purge_sys.rseg->latch};
@@ -636,7 +636,7 @@ TRANSACTIONAL_TARGET static void trx_purge_truncate_history()
       if (rseg.space != &space)
         continue;
 #ifdef SUX_LOCK_GENERIC
-      rseg.latch.rd_lock();
+      rseg.latch.rd_lock(SRW_LOCK_CALL);
 #else
       transactional_shared_lock_guard<srw_spin_lock> g{rseg.latch};
 #endif

storage/maria/s3_func.h

@@ -141,7 +141,6 @@ C_MODE_END
 C_MODE_START
 /* Dummy structures and interfaces to be used when compiling without S3 */
 struct s3_info;
-typedef struct s3_info S3_INFO;
 struct ms3_st;
 C_MODE_END
 #endif /* WITH_S3_STORAGE_ENGINE */

storage/myisam/mi_info.c

@@ -87,7 +87,10 @@ int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag)
     x->index_file_name  = share->index_file_name;
   }
   if ((flag & HA_STATUS_TIME) && !mysql_file_fstat(info->dfile, &state, MYF(0)))
+  {
+    MSAN_STAT_WORKAROUND(&state);
     x->update_time=state.st_mtime;
+  }
   else
     x->update_time=0;
   if (flag & HA_STATUS_AUTO)

strings/my_vsnprintf.c

@@ -738,10 +738,11 @@ size_t my_vsnprintf_ex(CHARSET_INFO *cs, char *to, size_t n,
     }
     else if (*fmt == 'f' || *fmt == 'g')
     {
+      double d;
 #if __has_feature(memory_sanitizer)         /* QQ: MSAN has double trouble? */
       __msan_check_mem_is_initialized(ap, sizeof(double));
 #endif
-      double d= va_arg(ap, double);
+      d= va_arg(ap, double);
 #if __has_feature(memory_sanitizer)        /* QQ: MSAN has double trouble? */
       __msan_unpoison(&d, sizeof(double));
 #endif

tpool/aio_liburing.cc

@@ -161,6 +161,8 @@ private:
       }
 
       io_uring_cqe_seen(&aio->uring_, cqe);
+      if (iocb->m_ret_len != iocb->m_len && !iocb->m_err)
+        finish_synchronous(iocb);
 
       // If we need to resubmit the IO operation, but the ring is full,
       // we will follow the same path as for any other error codes.

tpool/aio_linux.cc

@@ -128,6 +128,8 @@ class aio_linux final : public aio
           {
             iocb->m_ret_len= event.res;
             iocb->m_err= 0;
+            if (iocb->m_ret_len != iocb->m_len)
+              finish_synchronous(iocb);
           }
           iocb->m_internal_task.m_func= iocb->m_callback;
           iocb->m_internal_task.m_arg= iocb;

tpool/aio_simulated.cc

@@ -136,32 +136,7 @@ public:
   static void simulated_aio_callback(void *param)
   {
     aiocb *cb= (aiocb *) param;
-#ifdef _WIN32
+    synchronous(cb);
-    size_t ret_len;
-#else
-    ssize_t ret_len;
-#endif
-    int err= 0;
-    switch (cb->m_opcode)
-    {
-    case aio_opcode::AIO_PREAD:
-      ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
-      break;
-    case aio_opcode::AIO_PWRITE:
-      ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
-      break;
-    default:
-      abort();
-    }
-#ifdef _WIN32
-    if (static_cast<int>(ret_len) < 0)
-      err= GetLastError();
-#else
-    if (ret_len < 0)
-      err= errno;
-#endif
-    cb->m_ret_len = ret_len;
-    cb->m_err = err;
     cb->m_internal_task.m_func= cb->m_callback;
     thread_pool *pool= (thread_pool *)cb->m_internal;
     pool->submit_task(&cb->m_internal_task);

tpool/tpool.h

@@ -161,7 +161,7 @@ class aio
 {
 public:
   /**
-    Submit asyncronous IO.
+    Submit asynchronous IO.
     On completion, cb->m_callback is executed.
   */
   virtual int submit_io(aiocb *cb)= 0;
@@ -170,6 +170,10 @@ public:
   /** "Unind" file to AIO handler (used on Windows only) */
   virtual int unbind(const native_file_handle &fd)= 0;
   virtual ~aio(){};
+protected:
+  static void synchronous(aiocb *cb);
+  /** finish a partial read/write callback synchronously */
+  static void finish_synchronous(aiocb *cb);
 };
 
 class timer

tpool/tpool_generic.cc

@@ -51,6 +51,58 @@ namespace tpool
   static const std::chrono::milliseconds LONG_TASK_DURATION = std::chrono::milliseconds(500);
   static const int  OVERSUBSCRIBE_FACTOR = 2;
 
+/**
+  Process the cb synchronously
+*/
+void aio::synchronous(aiocb *cb)
+{
+#ifdef _WIN32
+  size_t ret_len;
+#else
+  ssize_t ret_len;
+#endif
+  int err= 0;
+  switch (cb->m_opcode)
+  {
+  case aio_opcode::AIO_PREAD:
+    ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
+    break;
+  case aio_opcode::AIO_PWRITE:
+    ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset);
+    break;
+  default:
+    abort();
+  }
+#ifdef _WIN32
+  if (static_cast<int>(ret_len) < 0)
+    err= GetLastError();
+#else
+  if (ret_len < 0)
+  {
+    err= errno;
+    ret_len= 0;
+  }
+#endif
+  cb->m_ret_len = ret_len;
+  cb->m_err = err;
+  if (!err && cb->m_ret_len != cb->m_len)
+    finish_synchronous(cb);
+}
+
+
+/**
+  A partial read/write has occured, continue synchronously.
+*/
+void aio::finish_synchronous(aiocb *cb)
+{
+  assert(cb->m_ret_len != (unsigned int) cb->m_len && !cb->m_err);
+  /* partial read/write */
+  cb->m_buffer= (char *) cb->m_buffer + cb->m_ret_len;
+  cb->m_len-= (unsigned int) cb->m_ret_len;
+  cb->m_offset+= cb->m_ret_len;
+  synchronous(cb);
+}
+
 /**
   Implementation of generic threadpool.
   This threadpool consists of the following components