MDEV-23730 s3.replication_partition 'innodb,mix' segv

This failure was caused because of several bugs:
- Someone had removed s3-slave-ignore-updates=1 from slave.cnf, which
  caused the slave to remove files that the master was working on.
- Bug in ha_partition::change_partitions() that didn't reset m_new_file
  in case of errors. This caused crashes in ha_maria::extra() as the
  maria handler was called on files that was already closed.
- In ma_pagecache there was a bug that when one got a read error one a
  big block (s3 block), it left the flag PCBLOCK_BIG_READ on for the page
  which cased an assert when the page where flushed.
- Flush all cached tables in case of ignored ALTER TABLE

Note that when merging code from 10.3, that fixes the partition bug, use
the code from this patch instead.

Changes to ma_pagecache.cc written or reviewed by Sanja

mysql-test/suite/s3/disabled.def

@@ -1,3 +0,0 @@
-replication_partition : MDEV-23730: Server crashes in ha_maria::extra
-replication_mixed     : MDEV-23770: Replication failure
-replication_stmt      : MDEV-23770: Replication failure

mysql-test/suite/s3/replication_partition.result

@@ -89,10 +89,18 @@ select sum(c1)+0 from t1;
 sum(c1)+0
 600
 stop slave;
-flush tables;
-select sum(c1)+0 from t1;
-sum(c1)+0
-600
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(11) NOT NULL,
+  `c2` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`)
+) ENGINE=S3 DEFAULT CHARSET=latin1
+ PARTITION BY RANGE (`c1`)
+(PARTITION `p1` VALUES LESS THAN (200) ENGINE = S3,
+ PARTITION `p2` VALUES LESS THAN (300) ENGINE = S3,
+ PARTITION `p3` VALUES LESS THAN (400) ENGINE = S3,
+ PARTITION `p4` VALUES LESS THAN (500) ENGINE = S3)
 connection master;
 drop table t1;
 connection slave;
@@ -204,8 +212,6 @@ slave-bin.000001	#	Query	#	#	use `database`; alter table t1 engine=S3
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
 slave-bin.000001	#	Query	#	#	use `database`; set @@sql_if_exists=1; ALTER TABLE t1 ADD PARTITION (PARTITION p4 VALUES LESS THAN (500))
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
-slave-bin.000001	#	Query	#	#	use `database`; flush tables
-slave-bin.000001	#	Gtid	#	#	GTID #-#-#
 slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t1` /* generated by server */
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
 slave-bin.000001	#	Query	#	#	use `database`; CREATE TABLE t2 (

mysql-test/suite/s3/replication_partition.test

@@ -7,6 +7,12 @@
 --source create_database.inc
 
 sync_slave_with_master;
+
+if (`select @@s3_slave_ignore_updates <> 1`)
+{
+  die "Slave is not configured with s3-slave-ignore-updates=1";
+}
+
 let $MYSQLD_DATADIR= `select @@datadir`;
 --replace_result $database database
 --eval use $database
@@ -65,20 +71,8 @@ sync_slave_with_master;
 select sum(c1)+0 from t1;
 stop slave;
 
-# .frm amd .par files should not exists on the salve as it has just seen the
-# ALTER TABLE which cased the removal of the .frm and .par files. The table
-# from the above "select sum()" came from table cache and was used as it's
-# id matches the one in S3
---error 1
---file_exists $MYSQLD_DATADIR/$database/t1.frm
---error 1
---file_exists $MYSQLD_DATADIR/$database/t1.par
-# Flushing the table cache will force the .frm and .par files to be
-# re-generated
-flush tables;
-select sum(c1)+0 from t1;
---file_exists $MYSQLD_DATADIR/$database/t1.frm
---file_exists $MYSQLD_DATADIR/$database/t1.par
+# Ensure the slave is using the new table
+show create table t1;
 
 connection master;
 drop table t1;

mysql-test/suite/s3/slave.cnf

@@ -2,6 +2,11 @@
 plugin-maturity = alpha
 plugin-load-add=@ENV.HA_S3_SO
 s3=ON
+s3-slave-ignore-updates=1
+
+# You can change the following when running the tests against
+# your own S3 setup
+
 #s3-host-name=s3.amazonaws.com
 #s3-protocol-version=Amazon
 #s3-bucket=MariaDB