Merge 10.11 into 11.4

extra/mariabackup/xtrabackup.cc

@@ -1226,7 +1226,8 @@ static void backup_file_op_fail(uint32_t space_id, int type,
 
 static void backup_undo_trunc(uint32_t space_id)
 {
-  undo_trunc_ids.insert(space_id);
+  if (space_id)
+    undo_trunc_ids.insert(space_id);
 }
 
 /* Function to store the space id of page0 INIT_PAGE

mysql-test/suite/encryption/r/recovery_memory.result

@@ -0,0 +1,15 @@
+CREATE TABLE t1(f1 text, index idx(f1(20))) ENGINE INNODB;
+set global innodb_fast_shutdown=0;
+# restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0
+set global debug_dbug="+d,ib_log_checkpoint_avoid_hard";
+INSERT INTO t1 SELECT repeat('a', 8000) FROM seq_1_to_1280;
+DELETE FROM t1;
+SET GLOBAL innodb_max_purge_lag_wait=0;
+INSERT INTO t1 VALUES('a');
+# XTRABACKUP PREPARE
+# restart
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+1
+ALTER TABLE t1 FORCE;
+DROP TABLE t1;

mysql-test/suite/encryption/t/recovery_memory.opt

@@ -0,0 +1,8 @@
+--innodb_doublewrite=0
+--innodb_log_file_size=24m
+--innodb_immediate_scrub_data_uncompressed=1
+--plugin-load-add=$FILE_KEY_MANAGEMENT_SO
+--loose-file-key-management
+--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt
+--file-key-management-encryption-algorithm=aes_cbc
+--innodb-encrypt-log=1

mysql-test/suite/encryption/t/recovery_memory.test

@@ -0,0 +1,45 @@
+--source include/have_debug.inc
+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source filekeys_plugin.inc
+
+let $basedir=$MYSQLTEST_VARDIR/tmp/backup;
+let MYSQLD_DATADIR=`select @@datadir`;
+
+CREATE TABLE t1(f1 text, index idx(f1(20))) ENGINE INNODB;
+
+# No checkpoint happens during this restart
+
+let $shutdown_timeout=;
+set global innodb_fast_shutdown=0;
+let $restart_parameters=--debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0;
+--source include/restart_mysqld.inc
+set global debug_dbug="+d,ib_log_checkpoint_avoid_hard";
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$basedir;
+--enable_result_log
+
+INSERT INTO t1 SELECT repeat('a', 8000) FROM seq_1_to_1280;
+DELETE FROM t1;
+SET GLOBAL innodb_max_purge_lag_wait=0;
+INSERT INTO t1 VALUES('a');
+
+--echo # XTRABACKUP PREPARE
+exec $XTRABACKUP --prepare --target-dir=$basedir;
+
+let $shutdown_timeout=0;
+--source include/shutdown_mysqld.inc
+
+# Since there is no checkpoint during previous run, we can
+# Copy the datafile from t1.ibd and start the server
+
+remove_file $MYSQLD_DATADIR/test/t1.ibd;
+copy_file $basedir/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd;
+--enable_result_log
+let $restart_parameters=;
+--source include/start_mysqld.inc
+
+SELECT COUNT(*) FROM t1;
+ALTER TABLE t1 FORCE;
+DROP TABLE t1;

storage/innobase/log/log0recv.cc

@@ -2418,7 +2418,7 @@ restart:
   ut_ad(log_sys.is_latest());
 
   alignas(8) byte iv[MY_AES_BLOCK_SIZE];
-  byte *decrypt_buf= storing == YES
+  byte *decrypt_buf= storing != BACKUP
     ? static_cast<byte*>(alloca(srv_page_size)) : nullptr;
 
   const lsn_t start_lsn{lsn};
@@ -2563,7 +2563,7 @@ restart:
         sql_print_warning("InnoDB: Ignoring malformed log record at LSN "
                           LSN_PF, lsn);
         /* the next record must not be same_page */
-        if (storing == YES) last_offset= 1;
+        if (storing != BACKUP) last_offset= 1;
         continue;
       }
       if (srv_operation == SRV_OPERATION_BACKUP)
@@ -2573,7 +2573,7 @@ restart:
                   lsn, b, l - recs + rlen, space_id, page_no));
       goto same_page;
     }
-    if (storing == YES) last_offset= 0;
+    if (storing != BACKUP) last_offset= 0;
     idlen= mlog_decode_varint_length(*l);
     if (UNIV_UNLIKELY(idlen > 5 || idlen >= rlen))
     {
@@ -2604,7 +2604,7 @@ restart:
       goto page_id_corrupted;
     l+= idlen;
     rlen-= idlen;
-    if (storing == YES)
+    if (storing != BACKUP)
     {
       mach_write_to_4(iv + 8, space_id);
       mach_write_to_4(iv + 12, page_no);
@@ -2654,15 +2654,15 @@ restart:
       ut_d(if ((b & 0x70) == INIT_PAGE || (b & 0x70) == OPTION)
              freed.erase(id));
       ut_ad(freed.find(id) == freed.end());
-      const byte *cl= storing == NO ? nullptr : l.ptr;
+      const byte *cl= nullptr; /* avoid bogus -Wmaybe-uninitialized */
       switch (b & 0x70) {
       case FREE_PAGE:
         ut_ad(freed.emplace(id).second);
         /* the next record must not be same_page */
-        if (storing == YES) last_offset= 1;
+        if (storing != BACKUP) last_offset= 1;
         goto free_or_init_page;
       case INIT_PAGE:
-        if (storing == YES) last_offset= FIL_PAGE_TYPE;
+        if (storing != BACKUP) last_offset= FIL_PAGE_TYPE;
       free_or_init_page:
         if (storing == BACKUP)
           continue;
@@ -2696,58 +2696,85 @@ restart:
           erase(r);
           continue;
         }
-      copy_if_needed:
         cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen);
         break;
       case EXTENDED:
-        if (storing != YES)
+        if (storing == NO)
+          /* We really only care about WRITE records to page 0, to
+          invoke fil_space_set_recv_size_and_flags().  As of now, the
+          EXTENDED records refer to index or undo log pages (which
+          page 0 never can be), or we have the TRIM_PAGES subtype for
+          shrinking a tablespace, to a larger number of pages than 0.
+          Either way, we can ignore this record during the preparation
+          for multi-batch recovery. */
           continue;
         if (UNIV_UNLIKELY(!rlen))
           goto record_corrupted;
+        if (storing == BACKUP)
+        {
+          if (rlen == 1 && undo_space_trunc)
+          {
+            cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen);
+            if (*cl == TRIM_PAGES)
+              undo_space_trunc(space_id);
+          }
+          continue;
+        }
+
         cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen);
         if (rlen == 1 && *cl == TRIM_PAGES)
         {
-          if (storing == BACKUP)
+          if (srv_is_undo_tablespace(space_id))
           {
-            if (space_id && undo_space_trunc)
+            if (page_no != SRV_UNDO_TABLESPACE_SIZE_IN_PAGES)
-              undo_space_trunc(space_id);
-          }
-          else if (srv_is_undo_tablespace(space_id))
-	  {
-	    if (page_no != SRV_UNDO_TABLESPACE_SIZE_IN_PAGES)
               goto record_corrupted;
             /* The entire undo tablespace will be reinitialized by
             innodb_undo_log_truncate=ON. Discard old log for all
-	    pages. */
+            pages. */
-	    trim({space_id, 0}, start_lsn);
+            trim({space_id, 0}, start_lsn);
-	    truncated_undo_spaces[space_id - srv_undo_space_id_start]=
+            truncated_undo_spaces[space_id - srv_undo_space_id_start]=
               { start_lsn, page_no};
-	  }
+          }
-	  else if (space_id != 0) goto record_corrupted;
+          else if (space_id != 0) goto record_corrupted;
-	  else
+          else
-	  {
+          {
-	    /* Shrink the system tablespace */
+            /* Shrink the system tablespace */
             trim({space_id, page_no}, start_lsn);
             truncated_sys_space= {start_lsn, page_no};
-	  }
+          }
           static_assert(UT_ARR_SIZE(truncated_undo_spaces) ==
                         TRX_SYS_MAX_UNDO_SPACES, "compatibility");
           /* the next record must not be same_page */
-          if (storing == YES) last_offset= 1;
+          last_offset= 1;
           continue;
         }
-        if (storing == YES) last_offset= FIL_PAGE_TYPE;
+        /* This record applies to an undo log or index page, and it
+        may be followed by subsequent WRITE or similar records for the
+        same page in the same mini-transaction. */
+        last_offset= FIL_PAGE_TYPE;
         break;
       case OPTION:
-        if (storing == YES && rlen == 5 && *l == OPT_PAGE_CHECKSUM)
+        /* OPTION records can be safely ignored in recovery */
-          goto copy_if_needed;
+        if (storing == YES &&
+            rlen == 5/* OPT_PAGE_CHECKSUM and CRC-32C; see page_checksum() */)
+        {
+          cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen);
+          if (*cl == OPT_PAGE_CHECKSUM)
+            break;
+        }
         /* fall through */
       case RESERVED:
         continue;
       case WRITE:
       case MEMMOVE:
       case MEMSET:
-        if (storing != YES)
+        if (storing == BACKUP)
+          continue;
+        if (storing == NO && UNIV_LIKELY(page_no != 0))
+          /* fil_space_set_recv_size_and_flags() is mandatory for storing==NO.
+          It is only applicable to page_no == 0. Other than that, we can just
+          ignore the payload and only compute the mini-transaction checksum;
+          there will be a subsequent call with storing==YES. */
           continue;
         if (UNIV_UNLIKELY(rlen == 0 || last_offset == 1))
           goto record_corrupted;
@@ -2789,7 +2816,7 @@ restart:
                                    last_offset)
                 : file_name_t::initial_flags;
               if (it == recv_spaces.end())
-                ut_ad(space_id == TRX_SYS_SPACE ||
+                ut_ad(storing == NO || space_id == TRX_SYS_SPACE ||
                       srv_is_undo_tablespace(space_id));
               else if (!it->second.space)
               {