BUG#49978: Replication tests don't clean up replication state at the end

Major replication test framework cleanup. This does the following:
 - Ensure that all tests clean up the replication state when they
   finish, by making check-testcase check the output of SHOW SLAVE STATUS.
   This implies:
    - Slave must not be running after test finished. This is good
      because it removes the risk for sporadic errors in subsequent
      tests when a test forgets to sync correctly.
    - Slave SQL and IO errors must be cleared when test ends. This is
      good because we will notice if a test gets an unexpected error in
      the slave threads near the end.
    - We no longer have to clean up before a test starts.
 - Ensure that all tests that wait for an error in one of the slave
   threads waits for a specific error. It is no longer possible to
   source wait_for_slave_[sql|io]_to_stop.inc when there is an error
   in one of the slave threads. This is good because:
    - If a test expects an error but there is a bug that causes
      another error to happen, or if it stops the slave thread without
      an error, then we will notice.
    - When developing tests, wait_for_*_to_[start|stop].inc will fail
      immediately if there is an error in the relevant slave thread.
      Before this patch, we had to wait for the timeout.
 - Remove duplicated and repeated code for setting up unusual replication
   topologies. Now, there is a single file that is capable of setting
   up arbitrary topologies (include/rpl_init.inc, but
   include/master-slave.inc is still available for the most common
   topology). Tests can now end with include/rpl_end.inc, which will clean
   up correctly no matter what topology is used. The topology can be
   changed with include/rpl_change_topology.inc.
 - Improved debug information when tests fail. This includes:
    - debug info is printed on all servers configured by include/rpl_init.inc
    - User can set $rpl_debug=1, which makes auxiliary replication files
      print relevant debug info.
 - Improved documentation for all auxiliary replication files. Now they
   describe purpose, usage, parameters, and side effects.
 - Many small code cleanups:
    - Made have_innodb.inc output a sensible error message.
    - Moved contents of rpl000017-slave.sh into rpl000017.test
    - Added mysqltest variables that expose the current state of
      disable_warnings/enable_warnings and friends.
    - Too many to list here: see per-file comments for details.

mysql-test/include/sync_slave_io_with_master.inc

@@ -2,35 +2,49 @@
 #
 # Waits until the slave IO thread has been synced, i.e., all events
 # have been copied over to slave.  Does not care if the SQL thread is
-# in sync.
+# in sync (or even running).
 #
 #
 # ==== Usage ====
 #
-# source include/sync_slave_io_with_master.inc;
+# [--let $sync_slave_connection= <connection_name>]
+# [--let $slave_timeout= NUMBER]
+# [--let $rpl_debug= 1]
+# --source include/sync_slave_io_with_master.inc
 #
-# Syncs to the current position on master, as found by SHOW MASTER
-# STATUS.
+# Syncs slave to the current position on master, as found by SHOW
+# MASTER STATUS.
 #
 # Must be called on the master. Will change connection to the slave.
 #
-# Parameters to this macro are $slave_timeout and
-# $master_connection. See wait_for_slave_param.inc for
-# descriptions.
+# Parameters:
+#   $sync_slave_connection
+#     By default, this script switches connection to 'slave'. If
+#     $sync_slave_connection is set, then '$sync_slave_connection' is
+#     used instead of 'slave'.
+#
+#   $slave_timeout
+#      See include/wait_for_slave_param.inc.
+#
+#   $rpl_debug
+#      See include/rpl_init.inc
 
-let $_master_file= query_get_value("SHOW MASTER STATUS", File, 1);
-let $_master_pos= query_get_value("SHOW MASTER STATUS", Position, 1);
+--let $include_filename= sync_slave_io_with_master.inc
+--source include/begin_include_file.inc
 
-connection slave;
 
-let $slave_error_message= Failed while waiting for slave IO thread to sync;
+--source include/save_master_pos.inc
 
-let $slave_param= Master_Log_File;
-let $slave_param_value= $_master_file;
-source include/wait_for_slave_param.inc;
+--let $rpl_connection_name= slave
+if (`SELECT '$sync_slave_connection' != ''`)
+{
+  --let $rpl_connection_name= $sync_slave_connection
+}
+--source include/rpl_connection.inc
 
-let $slave_param= Read_Master_Log_Pos;
-let $slave_param_value= $_master_pos;
-source include/wait_for_slave_param.inc;
+--source include/sync_io_with_master.inc
 
-let $slave_error_message= ;
+
+--let $include_filename= sync_slave_io_with_master.inc
+--let $skip_restore_connection= 1
+--source include/end_include_file.inc