content: /**
   * 
   */
  package com.taobao.tfstest;
  
  import org.junit.After;
  import org.junit.Assert;
  import org.junit.Before;
  import org.junit.Test;
  
  import com.taobao.gaia.KillTypeEnum;
  
  /**
   * @author Administrator
   *
   */
  public class Failover_ha_test extends FailOverBaseCase {
  	
  	//@Test
  	public void Failover_XX_ns_restart_without_meta_data(){
  		/* Do nothing */
  		return ;
  	}
  
  	//@Test
  	public void Failover_01_ns_ha_switch_once(){
  		
  		boolean bRet = false;
  		caseName = "Failover_01_ns_ha_switch_once";
  		log.info(caseName + "===> start");
  		
  		/* Set loop flag */
  		bRet = setSeedFlag(LOOPON);
  		Assert.assertTrue(bRet);
  		
  		/* Set seed size */
  		bRet = setSeedSize(1);
  		Assert.assertTrue(bRet);
  		
  		/* Set unlink ratio */
  		bRet = setUnlinkRatio(50);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  	
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Restart ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	//@Test
  	public void Failover_02_ns_ha_switch_serval_times(){
  		
  		boolean bRet = false;
  		caseName = "Failover_02_ns_ha_switch_serval_times";
  		log.info(caseName + "===> start");
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killMasterNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Restart ns */
  			bRet = startSlaveNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Wait 20s for recover */
  		sleep (10);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	//@Test
  	public void Failover_03_ns_ha_slave_restart_once(){
  		
  		boolean bRet = false;
  		caseName = "Failover_03_ns_ha_slave_restart_once";
  		log.info(caseName + "===> start");
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Start slave ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  
  	public void Failover_04_ns_ha_slave_restart_serval_times(){
  		
  		boolean bRet = false;
  		caseName = "Failover_04_ns_ha_slave_restart_serval_times";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  	
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killSlaveNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Start slave ns */
  			bRet = startSlaveNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Wait 20s for recover */
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	@Test
  	public void Failover_05_ns_ha_ns_restart_once(){
  		
  		boolean bRet = false;
  		caseName = "Failover_05_ns_ha_ns_restart_once";
  		log.info(caseName + "===> start");
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		sleep(20);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Start slave ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 60s for recover */
  		sleep (60);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copy count */
  		bRet = chkBlockCopyCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_06_ns_ha_ns_restart_serval_times(){
  		
  		boolean bRet = false;
  		caseName = "Failover_06_ns_ha_ns_restart_serval_times";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killMasterNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Kill master ns */
  			bRet = killMasterNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Start slave ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 60s for recover */
  		sleep (60);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_07_ns_ha_slave_down(){
  		
  		boolean bRet = false;
  		caseName = "Failover_07_ns_ha_slave_down";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  public void Failover_08_one_ds_out_copy_block_switch(){
  		
  		boolean bRet = false;
  		caseName = "Failover_08_one_ds_out_copy_block_switch";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill one ds */
  		bRet = killOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  
  	public void Failover_09_one_ds_in_switch(){
  		
  		boolean bRet = false;
  		caseName = "Failover_09_one_ds_in_switch";
  		log.info(caseName + "===> start");
  		
  		/* Kill one ds */
  		bRet = killOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (30);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* One ds in */
  		bRet = startOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_10_compact_switch(){
  		
  		boolean bRet = false;
  		caseName = "Failover_10_compact_switch";
  		log.info(caseName + "===> start");
  		
  		/* Set unlink ratio */
  		bRet = setUnlinkRatio(70);
  		Assert.assertTrue(bRet);	
  		
  		/* Start write cmd */
  		bRet = writeCmd();
  		Assert.assertTrue(bRet);
  		
  		sleep (400);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Wait for compact */
  		sleep(300);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateRun(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the final rate */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Set unlink ratio */
  		bRet = setUnlinkRatio(50);
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_11_ns_ha_switch_once_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_11_ns_ha_switch_once_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Restart ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_12_ns_ha_switch_serval_times_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_12_ns_ha_switch_serval_times_without_press";
  		log.info(caseName + "===> start");
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killMasterNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Restart ns */
  			bRet = startSlaveNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Wait 20s for recover */
  		sleep (10);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_13_ns_ha_slave_restart_once_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_13_ns_ha_slave_restart_once_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Put the seed */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (10);
  		
  		/* Start slave ns */
  		bRet = startSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  
  	public void Failover_14_ns_ha_slave_restart_serval_times_whitout_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_14_ns_ha_slave_restart_serval_times_whitout_press";
  		log.info(caseName + "===> start");
  		
  		/* Put seed */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killSlaveNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Start slave ns */
  			bRet = startSlaveNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Wait 20s for recover */
  		sleep (10);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_15_ns_ha_ns_restart_once_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_15_ns_ha_ns_restart_once_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		sleep(20);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_16_ns_ha_ns_restart_serval_times_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_16_ns_ha_ns_restart_serval_times_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		for (int iLoop = 0; iLoop < 10; iLoop ++)
  		{
  			/* Kill master ns */
  			bRet = killSlaveNs();
  			Assert.assertTrue(bRet);
  			
  			sleep(20);
  			
  			/* Kill master ns */
  			bRet = killSlaveNs();
  			Assert.assertTrue(bRet);
  		}
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_17_ns_ha_slave_down_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_17_ns_ha_slave_down_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = putSeed();	
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killSlaveNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the rate of write process */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Check vip */
  		bRet = chkVip();
  		Assert.assertTrue(bRet);
  		
  		/* Check the status of servers */
  		bRet = chkAlive();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  public void Failover_18_one_ds_out_copy_block_switch_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_18_one_ds_out_copy_block_switch_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Write file */
  		bRet = putSeed();	
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Kill one ds */
  		bRet = killOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Wait 20s for recover */
  		sleep (20);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  
  	public void Failover_19_one_ds_in_switch_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_19_one_ds_in_switch_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Kill one ds */
  		bRet = killOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Write file */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		/* One ds in */
  		bRet = startOneDs();
  		Assert.assertTrue(bRet);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  	
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	public void Failover_20_compact_switch_without_press(){
  		
  		boolean bRet = false;
  		caseName = "Failover_20_compact_switch_without_press";
  		log.info(caseName + "===> start");
  		
  		/* Set unlink ratio */
  		bRet = setUnlinkRatio(70);
  		Assert.assertTrue(bRet);
  		
  		/* Start write cmd */
  		bRet = putSeed();
  		Assert.assertTrue(bRet);
  		
  		/* Wait for compact */
  		sleep(300);
  		
  		/* Kill master ns */
  		bRet = killMasterNs();
  		Assert.assertTrue(bRet);
  		
  		/* Check block copys */
  		bRet = chkBlockCntBothNormal(BLOCKCOPYCNT);
  		Assert.assertTrue(bRet);
  		
  		/* Stop write cmd */
  		bRet = writeCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Check the final rate */
  		bRet = checkRateEnd(SUCCESSRATE, WRITEONLY|READ|UNLINK);
  		Assert.assertTrue(bRet);
  		
  		/* Read file */
  		bRet = chkFinalRetSuc();
  		Assert.assertTrue(bRet);
  		
  		/* Set unlink ratio */
  		bRet = setUnlinkRatio(50);
  		Assert.assertTrue(bRet);
  		
  		log.info(caseName + "===> end");
  		return ;
  	}
  	
  	//@After
  	public void tearDown(){
  		boolean bRet = false;
  		
  		/* Stop all client process */
  		bRet = allCmdStop();
  		Assert.assertTrue(bRet);
  		
  		/* Move the seed file list */
  		bRet = mvSeedFile();
  		Assert.assertTrue(bRet);
  		
  		/* Clean the caseName */
  		caseName = "";
  	}
  	
  	@Before
  	public void setUp(){
  		boolean bRet = false;
  		
  		/* Reset case name */
  		caseName = "";
  		
  		/* Set the failcount */
  		bRet = setAllFailCnt();
  		Assert.assertTrue(bRet);
  		
  		/* Kill the grid */
  		bRet = tfsGrid.stop(KillTypeEnum.FORCEKILL, WAITTIME);
  		Assert.assertTrue(bRet);
  		
  		/* Set Vip */
  		bRet = migrateVip();
  		Assert.assertTrue(bRet);
  		
  		/* Clean the log file */
  		bRet = tfsGrid.clean();
  		Assert.assertTrue(bRet);
  		
  		bRet = tfsGrid.start();
  		Assert.assertTrue(bRet);
  		
  		/* Set failcount */
  		bRet = resetAllFailCnt();
  		Assert.assertTrue(bRet);
  	
  	}
  }
  