BUG#49978: Replication tests don't clean up replication state at the end

Major replication test framework cleanup. This does the following:
 - Ensure that all tests clean up the replication state when they
   finish, by making check-testcase check the output of SHOW SLAVE STATUS.
   This implies:
    - Slave must not be running after test finished. This is good
      because it removes the risk for sporadic errors in subsequent
      tests when a test forgets to sync correctly.
    - Slave SQL and IO errors must be cleared when test ends. This is
      good because we will notice if a test gets an unexpected error in
      the slave threads near the end.
    - We no longer have to clean up before a test starts.
 - Ensure that all tests that wait for an error in one of the slave
   threads waits for a specific error. It is no longer possible to
   source wait_for_slave_[sql|io]_to_stop.inc when there is an error
   in one of the slave threads. This is good because:
    - If a test expects an error but there is a bug that causes
      another error to happen, or if it stops the slave thread without
      an error, then we will notice.
    - When developing tests, wait_for_*_to_[start|stop].inc will fail
      immediately if there is an error in the relevant slave thread.
      Before this patch, we had to wait for the timeout.
 - Remove duplicated and repeated code for setting up unusual replication
   topologies. Now, there is a single file that is capable of setting
   up arbitrary topologies (include/rpl_init.inc, but
   include/master-slave.inc is still available for the most common
   topology). Tests can now end with include/rpl_end.inc, which will clean
   up correctly no matter what topology is used. The topology can be
   changed with include/rpl_change_topology.inc.
 - Improved debug information when tests fail. This includes:
    - debug info is printed on all servers configured by include/rpl_init.inc
    - User can set $rpl_debug=1, which makes auxiliary replication files
      print relevant debug info.
 - Improved documentation for all auxiliary replication files. Now they
   describe purpose, usage, parameters, and side effects.
 - Many small code cleanups:
    - Made have_innodb.inc output a sensible error message.
    - Moved contents of rpl000017-slave.sh into rpl000017.test
    - Added mysqltest variables that expose the current state of
      disable_warnings/enable_warnings and friends.
    - Too many to list here: see per-file comments for details.

mysql-test/include/wait_for_slave_io_error.inc

@@ -4,56 +4,86 @@
 # error, or until a timeout is reached. Also waits until the IO
 # thread has completely stopped.
 #
+#
 # ==== Usage ====
 #
-# # Wait several errors.  
-# let $slave_io_errno= 1, 2, 3; 
-# source include/wait_for_slave_io_error.inc;
-#
-# # Print error message 
-# let $slave_io_errno= 1; 
-# let $show_slave_io_error= 1; 
-# source include/wait_for_slave_io_error.inc;
+# --let $slave_io_errno= NUMBER [, NUMBER ...]
+# [--let $show_slave_io_error= 1]
+# [--let $slave_io_error_is_nonfatal= 1]
+# [--let $rpl_debug= 1]
+# [--let $slave_timeout= NUMBER]
+# --source include/wait_for_slave_io_error.inc
 #
 # Parameters:
+#   $slave_io_errno
+#     The expected IO error numbers. This can be either a single
+#     number, or a comma-separated list of numbers. Examples:
+#       --let $slave_io_errno= 1040, 1053, 2002, 2003, 2006, 2013
+#       --let $slave_io_errno= 1045
+#     (After BUG#41956 has been fixed, this will be required to be
+#     symbolic names instead of numbers.)
 #
-# $slave_io_errno
-#   The expected IO error numbers.  This is required.
-#   (After BUG#41956 has been fixed, this will be required to be a
-#   symbolic name instead of a numbers.)
+#   $show_slave_io_error
+#     If set, will print the error to the query log.
 #
-# $show_slave_io_error
-#   If set, will print the error to the query log.
+#   $slave_io_error_is_nonfatal
+#     By default, we wait for the slave IO thread to stop completely
+#     (i.e., until Slave_IO_State is empty).  If this variable is set,
+#     then we don't wait. This is useful if the error is non-fatal
+#     (e.g., temporary connection error) and does not cause the slave
+#     IO thread to stop.
 # 
-# $slave_timeout
-#   See wait_for_slave_param.inc for description.
+#   $slave_timeout
+#     See include/wait_for_slave_param.inc
 # 
-# $master_connection
-#   See wait_for_slave_param.inc for description.
+#   $rpl_debug
+#     See include/rpl_init.inc
+
+
+--let $include_filename= wait_for_slave_io_error.inc [errno=$slave_io_errno]
+--source include/begin_include_file.inc
 
-if (!$slave_io_errno) {
-  --die !!!ERROR IN TEST: you must set \$slave_io_errno before sourcing wait_for_slave_io_error.inc
-}
 
 let $old_slave_param_comparison= $slave_param_comparison;
 let $slave_param= Last_IO_Errno;
 let $slave_param_comparison= !=;
 let $slave_param_value= 0;
-let $slave_error_message= Failed while waiting for slave to produce an error in its sql thread;
 source include/wait_for_slave_param.inc;
-let $slave_error_message= ;
 let $slave_param_comparison= $old_slave_param_comparison;
 
-let $_error= query_get_value(SHOW SLAVE STATUS, Last_IO_Errno, 1);
-if (`SELECT $_error NOT IN ($slave_io_errno)`) {
-  --echo **** Slave stopped with wrong error code: $_error (expected $slave_io_errno) ****
+let $_wfsie_errno= query_get_value(SHOW SLAVE STATUS, Last_IO_Errno, 1);
+
+if (!$slave_io_errno) {
+  --echo !!!ERROR IN TEST: you must set \$slave_io_errno before you source
+  --echo !!!wait_for_slave_sql_error.inc. The error we got this time was '$_wfsie_errno',
+  --echo !!!so you probably want to add the following line to your test case:
+  --echo !!!  --let \$slave_io_errno= $_wfsie_errno
+  --die !!!ERROR IN TEST: you must set \$slave_io_errno before sourcing wait_for_slave_io_error.inc
+}
+
+if (`SELECT $_wfsie_errno NOT IN ($slave_io_errno)`) {
+  --echo **** Slave stopped with wrong error code: $_wfsie_errno (expected $slave_io_errno) ****
   source include/show_rpl_debug_info.inc;
-  --echo **** Slave stopped with wrong error code: $_error (expected $slave_io_errno) ****
+  --echo **** Slave stopped with wrong error code: $_wfsie_errno (expected $slave_io_errno) ****
   --die Slave stopped with wrong error code
 }
 
 if ($show_slave_io_error)
 {
-  let $error= query_get_value("SHOW SLAVE STATUS", Last_IO_Error, 1);
-  echo Last_IO_Error = $error;
+  --let $_wait_for_slave_io_error_old_status_items= $status_items
+  --let $status_items= Last_IO_Error
+  --source include/show_slave_status.inc
+  --let $status_items= $_wait_for_slave_io_error_old_status_items
 }
+
+if (!$slave_io_error_is_nonfatal)
+{
+  --let $slave_param= Slave_IO_State
+  --let $slave_param_value=
+  --source include/wait_for_slave_param.inc
+}
+--let $slave_io_error_is_nonfatal= 0
+
+
+--let $include_filename= wait_for_slave_io_error.inc [errno=$slave_io_errno]
+--source include/end_include_file.inc