support for three slaves in mysql-test

mysql-test/mysql-test-run.sh:
  start three slaves in failover test
mysql-test/r/rpl_failsafe.result:
  check two more slaves
mysql-test/t/rpl_failsafe.test:
  check two more slaves

mysql-test/mysql-test-run.sh

@@ -124,7 +124,7 @@ MASTER_RUNNING=0
 MASTER_MYPORT=9306
 SLAVE_RUNNING=0
 SLAVE_MYPORT=9307
-MYSQL_MANAGER_PORT=9308
+MYSQL_MANAGER_PORT=9305 # needs to be out of the way of slaves
 MYSQL_MANAGER_PW_FILE=$MYSQL_TEST_DIR/var/tmp/manager.pwd
 MYSQL_MANAGER_LOG=$MYSQL_TEST_DIR/var/log/manager.log
 MYSQL_MANAGER_USER=root
@@ -473,6 +473,13 @@ mysql_install_db () {
 	error "Could not install slave test DBs"
 	exit 1
     fi
+    
+    for slave_num in 1 2 ;
+    do
+     mkdir -p var/slave$slave_num-data/mysql
+     mkdir -p var/slave$slave_num-data/test
+     cp var/slave-data/mysql/* var/slave$slave_num-data/mysql
+    done
     # Give mysqld some time to die.
     sleep $SLEEP_TIME
     return 0
@@ -647,10 +654,33 @@ start_master()
 start_slave()
 {
     [ x$SKIP_SLAVE = x1 ] && return
-    [ x$SLAVE_RUNNING = 1 ] && return
+    eval "this_slave_running=\$SLAVE$1_RUNNING"
-    
+    [ x$this_slave_running = 1 ] && return
+    #when testing fail-safe replication, we will have more than one slave
+    #in this case, we start secondary slaves with an argument
+    slave_ident="slave$1"
+    if [ -n "$1" ] ;
+    then
+     slave_server_id=`$EXPR 2 + $1`
+     slave_rpl_rank=$slave_server_id
+     slave_port=`expr $SLAVE_MYPORT + $1`
+     slave_log="$SLAVE_MYLOG.$1"
+     slave_err="$SLAVE_MYERR.$1"
+     slave_datadir="var/$slave_ident-data/"
+     slave_pid="$MYRUN_DIR/mysqld-$slave_ident.pid"
+     slave_sock="$SLAVE_MYSOCK-$1"
+    else
+     slave_server_id=2
+     slave_rpl_rank=2
+     slave_port=$SLAVE_MYPORT 
+     slave_log=$SLAVE_MYLOG
+     slave_err=$SLAVE_MYERR
+     slave_datadir=$SLAVE_MYDDIR
+     slave_pid=$SLAVE_MYPID
+     slave_sock="$SLAVE_MYSOCK"
+   fi 
     # Remove stale binary logs
-    $RM -f $MYSQL_TEST_DIR/var/log/slave-bin.*
+    $RM -f $MYSQL_TEST_DIR/var/log/$slave_ident-bin.*
     
     #run slave initialization shell script if one exists
     if [ -f "$slave_init_script" ] ;
@@ -664,22 +694,22 @@ start_slave()
 	    --master-host=127.0.0.1 \
 	    --master-password= \
 	    --master-port=$MASTER_MYPORT \
-	    --server-id=2 --rpl-recovery-rank=2"
+	    --server-id=$slave_server_id --rpl-recovery-rank=$slave_rpl_rank"
    else
      master_info=$SLAVE_MASTER_INFO
    fi	    
     
-    $RM -f $SLAVE_MYDDIR/log.*	
+    $RM -f $slave_datadir/log.*	
     slave_args="--no-defaults $master_info \
     	    --exit-info=256 \
-	    --log-bin=$MYSQL_TEST_DIR/var/log/slave-bin \
+	    --log-bin=$MYSQL_TEST_DIR/var/log/$slave_ident-bin \
 	    --log-slave-updates \
-            --log=$SLAVE_MYLOG \
+            --log=$slave_log \
             --basedir=$MY_BASEDIR \
-            --datadir=$SLAVE_MYDDIR \
+            --datadir=$slave_datadir \
-	    --pid-file=$SLAVE_MYPID \
+	    --pid-file=$slave_pid \
-	    --port=$SLAVE_MYPORT \
+	    --port=$slave_port \
-	    --socket=$SLAVE_MYSOCK \
+	    --socket=$slave_sock \
 	    --character-sets-dir=$CHARSETSDIR \
 	    --default-character-set=$CHARACTER_SET \
 	    --core \
@@ -688,27 +718,27 @@ start_slave()
 	    --skip-innodb --skip-slave-start \
 	    --slave-load-tmpdir=$SLAVE_LOAD_TMPDIR \
 	    --report-host=127.0.0.1 --report-user=root \
-	    --report-port=$SLAVE_MYPORT \
+	    --report-port=$slave_port \
 	    --master-retry-count=5 \
 	     $SMALL_SERVER \
              $EXTRA_SLAVE_OPT $EXTRA_SLAVE_MYSQLD_OPT"
-    CUR_MYERR=$SLAVE_MYERR
+    CUR_MYERR=$slave_err
-    CUR_MYSOCK=$SLAVE_MYSOCK
+    CUR_MYSOCK=$slave_sock
   
     if [ x$DO_DDD = x1 ]
     then
       $ECHO "set args $master_args" > $GDB_SLAVE_INIT
-      manager_launch slave ddd -display $DISPLAY --debugger \
+      manager_launch $slave_ident ddd -display $DISPLAY --debugger \
        "gdb -x $GDB_SLAVE_INIT" $SLAVE_MYSQLD 
     elif [ x$DO_GDB = x1 ]
     then
       $ECHO "set args $slave_args" > $GDB_SLAVE_INIT
-      manager_launch slave $XTERM -display $DISPLAY -title "Slave" -e gdb -x \
+      manager_launch $slave_ident $XTERM -display $DISPLAY -title "Slave" -e gdb -x \
        $GDB_SLAVE_INIT $SLAVE_MYSQLD 
     else
-      manager_launch slave $SLAVE_MYSQLD $slave_args
+      manager_launch $slave_ident $SLAVE_MYSQLD $slave_args
     fi
-    SLAVE_RUNNING=1
+    eval "SLAVE$1_RUNNING=1"
 }
 
 mysql_start () {
@@ -721,23 +751,31 @@ mysql_start () {
 
 stop_slave ()
 {
-  if [ x$SLAVE_RUNNING = x1 ]
+  eval "this_slave_running=\$SLAVE$1_RUNNING"
+  slave_ident="slave$1"
+  if [ -n "$1" ] ;
   then
-    manager_term slave
+   slave_pid="$MYRUN_DIR/mysqld-$slave_ident.pid"
-    if [ $? != 0 ] && [ -f $SLAVE_MYPID ]
+  else
+   slave_pid=$SLAVE_MYPID
+  fi 
+  if [ x$this_slave_running = x1 ]
+  then
+    manager_term $slave_ident
+    if [ $? != 0 ] && [ -f $slave_pid ]
     then # try harder!
      $ECHO "slave not cooperating with mysqladmin, will try manual kill"
-     kill `$CAT $SLAVE_MYPID`
+     kill `$CAT $slave_pid`
      sleep $SLEEP_TIME
      if [ -f $SLAVE_MYPID ] ; then
        $ECHO "slave refused to die. Sending SIGKILL"
-       kill -9 `$CAT $SLAVE_MYPID`
+       kill -9 `$CAT $slave_pid`
-       $RM -f $SLAVE_MYPID
+       $RM -f $slave_pid
      else
       $ECHO "slave responded to SIGTERM " 
      fi
     fi
-    SLAVE_RUNNING=0
+    eval "SLAVE$1_RUNNING=0"
   fi  
 }
 
@@ -771,6 +809,8 @@ mysql_stop ()
  stop_master
  $ECHO "Master shutdown finished"
  stop_slave
+ stop_slave 1
+ stop_slave 2
  $ECHO "Slave shutdown finished"
  
  return 1
@@ -800,6 +840,7 @@ run_testcase ()
  slave_init_script=$TESTDIR/$tname-slave.sh
  slave_master_info_file=$TESTDIR/$tname-slave-master-info.opt
  SKIP_SLAVE=`$EXPR \( $tname : rpl \) = 0`
+ many_slaves=`$EXPR \( $tname : rpl_failsafe \) != 0`
  if [ -n "$SKIP_TEST" ] ; then 
    SKIP_THIS_TEST=`$EXPR \( $tname : "$SKIP_TEST" \) != 0`
    if [ x$SKIP_THIS_TEST = x1 ] ;
@@ -874,6 +915,10 @@ run_testcase ()
      stop_slave
      start_slave
    fi
+   if [ x$many_slaves = x1 ]; then
+    start_slave 1
+    start_slave 2
+   fi
  fi
  cd $MYSQL_TEST_DIR
   

mysql-test/r/rpl_failsafe.result

@@ -9,3 +9,9 @@ rpl_recovery_rank	1
 show variables like 'rpl_recovery_rank';
 Variable_name	Value
 rpl_recovery_rank	2
+show variables like 'rpl_recovery_rank';
+Variable_name	Value
+rpl_recovery_rank	3
+show variables like 'rpl_recovery_rank';
+Variable_name	Value
+rpl_recovery_rank	4

mysql-test/t/rpl_failsafe.test

@@ -1,5 +1,11 @@
 source include/master-slave.inc;
+connect (slave_sec,localhost,root,,test,0,slave.sock-1); 
+connect (slave_ter,localhost,root,,test,0,slave.sock-2); 
 connection master;
 show variables like 'rpl_recovery_rank';
 connection slave;
 show variables like 'rpl_recovery_rank';
+connection slave_sec;
+show variables like 'rpl_recovery_rank';
+connection slave_ter;
+show variables like 'rpl_recovery_rank';