MDEV-11288 Server crashes in Binlog_crypt_data::init trying to feed encrypted log without decryption capabilities

mysql-test/suite/binlog_encryption/disabled.def

@@ -1,4 +1,3 @@
-encrypted_master_switch_to_unencrypted : MDEV-11288 - server crash
 binlog_incident                        : MDEV-11319 - mysqlbinlog crash or failure
 encrypted_master_lost_key              : MDEV-11323 - unspecified behavior for IO thread
 rpl_checksum_cache                     : MDEV-11486 - sporadic failure in IO thread

mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result

@@ -0,0 +1,85 @@
+#################
+# Initialization
+#################
+include/rpl_init.inc [topology=1->2]
+connection server_2;
+include/stop_slave.inc
+#####################################################
+# Part 1: unencrypted master
+#####################################################
+connection server_1;
+call mtr.add_suppression("Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error;");
+CREATE TABLE table1_no_encryption (
+pk INT AUTO_INCREMENT PRIMARY KEY,
+ts TIMESTAMP NULL,
+b BLOB
+) ENGINE=MyISAM;
+INSERT INTO table1_no_encryption VALUES (NULL,NOW(),'data_no_encryption');
+INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption;
+FLUSH BINARY LOGS;
+SET binlog_format=ROW;
+INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption;
+INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption;
+NOT FOUND /table1_no_encryption/ in master-bin.0*
+#####################################################
+# Part 2: restart master, now with binlog encryption
+#####################################################
+connection default;
+connection server_1;
+CREATE TABLE table2_to_encrypt (
+pk INT AUTO_INCREMENT PRIMARY KEY,
+ts TIMESTAMP NULL,
+b BLOB
+) ENGINE=MyISAM;
+INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt');
+INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt;
+FLUSH BINARY LOGS;
+SET binlog_format=ROW;
+INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt;
+INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt;
+NOT FOUND /table2_to_encrypt/ in master-bin.0*
+#####################################################
+# Part 3: restart master again without encryption
+#####################################################
+connection default;
+connection server_1;
+CREATE TABLE table3_no_encryption (
+pk INT AUTO_INCREMENT PRIMARY KEY,
+ts TIMESTAMP NULL,
+b BLOB
+) ENGINE=MyISAM;
+INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption');
+INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption;
+INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption;
+#####################################################
+# Check: resume replication and check how it goes
+#####################################################
+connection server_2;
+start slave;
+include/wait_for_slave_io_error.inc [errno=1236]
+connection server_2;
+connection server_2;
+connection server_2;
+SHOW TABLES;
+Tables_in_test
+table1_no_encryption
+include/stop_slave.inc
+reset slave;
+##########
+# Cleanup
+##########
+connection server_1;
+reset master;
+SELECT COUNT(*) FROM table1_no_encryption;
+COUNT(*)
+8
+SELECT COUNT(*) FROM table2_to_encrypt;
+COUNT(*)
+8
+SELECT COUNT(*) FROM table3_no_encryption;
+COUNT(*)
+4
+DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption;
+connection server_2;
+include/start_slave.inc
+include/rpl_end.inc

mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test

@@ -35,6 +35,8 @@
 
 --connection server_1
 
+call mtr.add_suppression("Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error;");
+
 CREATE TABLE table1_no_encryption (
   pk INT AUTO_INCREMENT PRIMARY KEY,
   ts TIMESTAMP NULL,
@@ -107,29 +109,31 @@ INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption;
 --echo #####################################################
 --connection server_2
 
---disable_connect_log
+start slave;
---source include/start_slave.inc
+--let slave_io_errno=1236
---enable_connect_log
+--source include/wait_for_slave_io_error.inc
---sync_with_master
 
 --sorted_result
 SHOW TABLES;
 
+--disable_connect_log
+--source include/stop_slave.inc
+--enable_connect_log
+reset slave;
+
 --echo ##########
 --echo # Cleanup
 --echo ##########
 
 --connection server_1
+reset master;
 
 SELECT COUNT(*) FROM table1_no_encryption;
 SELECT COUNT(*) FROM table2_to_encrypt;
 SELECT COUNT(*) FROM table3_no_encryption;
 DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption;
 
---save_master_pos
-
 --connection server_2
---sync_with_master
-
 --disable_connect_log
+--source include/start_slave.inc
 --source include/rpl_end.inc

sql/encryption.cc

@@ -29,6 +29,10 @@ uint no_key(uint)
 {
   return ENCRYPTION_KEY_VERSION_INVALID;
 }
+uint zero_size(uint,uint)
+{
+  return 0;
+}
 
 static int ctx_init(void *ctx, const unsigned char* key, unsigned int klen,
                     const unsigned char* iv, unsigned int ivlen, int flags,
@@ -97,6 +101,7 @@ int finalize_encryption_plugin(st_plugin_int *plugin)
   encryption_handler.encryption_key_get_func=
       (uint (*)(uint, uint, uchar*, uint*))no_key;
   encryption_handler.encryption_key_get_latest_version_func= no_key;
+  encryption_handler.encryption_ctx_size_func= zero_size;
 
   if (plugin && plugin->plugin->deinit && plugin->plugin->deinit(NULL))
   {

sql/sql_repl.cc

@@ -2212,6 +2212,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
   THD *thd= info->thd;
   String *packet= info->packet;
   Log_event_type event_type;
+  DBUG_ENTER("send_format_descriptor_event");
 
   /**
    * 1) reset fdev before each log-file
@@ -2226,12 +2227,12 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
   {
     info->errmsg= "Out of memory initializing format_description event";
     info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG;
-    return 1;
+    DBUG_RETURN(1);
   }
 
   /* reset transmit packet for the event read from binary log file */
   if (reset_transmit_packet(info, info->flags, &ev_offset, &info->errmsg))
-    return 1;
+    DBUG_RETURN(1);
 
   /*
     Try to find a Format_description_log_event at the beginning of
@@ -2247,7 +2248,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
   if (error)
   {
     set_read_error(info, error);
-    return 1;
+    DBUG_RETURN(1);
   }
 
   event_type= (Log_event_type)((uchar)(*packet)[LOG_EVENT_OFFSET+ev_offset]);
@@ -2268,7 +2269,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
     sql_print_warning("Failed to find format descriptor event in "
                       "start of binlog: %s",
                       info->log_file_name);
-    return 1;
+    DBUG_RETURN(1);
   }
 
   info->current_checksum_alg= get_checksum_alg(packet->ptr() + ev_offset,
@@ -2288,7 +2289,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
     sql_print_warning("Master is configured to log replication events "
                       "with checksum, but will not send such events to "
                       "slaves that cannot process them");
-    return 1;
+    DBUG_RETURN(1);
   }
 
   uint ev_len= packet->length() - ev_offset;
@@ -2302,7 +2303,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
     info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG;
     info->errmsg= "Corrupt Format_description event found "
         "or out-of-memory";
-    return 1;
+    DBUG_RETURN(1);
   }
   delete info->fdev;
   info->fdev= tmp;
@@ -2361,7 +2362,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
   {
     info->errmsg= "Failed on my_net_write()";
     info->error= ER_UNKNOWN_ERROR;
-    return 1;
+    DBUG_RETURN(1);
   }
 
   /*
@@ -2381,7 +2382,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
   if (error)
   {
     set_read_error(info, error);
-    return 1;
+    DBUG_RETURN(1);
   }
 
   event_type= (Log_event_type)((uchar)(*packet)[LOG_EVENT_OFFSET]);
@@ -2393,14 +2394,15 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
     if (!sele)
     {
       info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG;
-      return 1;
+      DBUG_RETURN(1);
     }
 
     if (info->fdev->start_decryption(sele))
     {
       info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG;
       info->errmsg= "Could not decrypt binlog: encryption key error";
-      return 1;
+      delete sele;
+      DBUG_RETURN(1);
     }
     delete sele;
   }
@@ -2416,7 +2418,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log,
 
 
   /** all done */
-  return 0;
+  DBUG_RETURN(0);
 }
 
 static bool should_stop(binlog_send_info *info)