/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.activemq.transport.failover;

import org.apache.activemq.broker.PublishedAddressPolicy;
import org.apache.activemq.broker.TransportConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
 * Complex cluster test that will exercise the dynamic failover capabilities of
 * a network of brokers. Using a networking of 3 brokers where the 3rd broker is
 * removed and then added back in it is expected in each test that the number of
 * connections on the client should start with 3, then have two after the 3rd
 * broker is removed and then show 3 after the 3rd broker is reintroduced.
 */
public class FailoverComplexClusterTest extends FailoverClusterTestSupport {
    protected final Logger LOG = LoggerFactory.getLogger(FailoverComplexClusterTest.class);

    private static final String BROKER_A_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61616";
    private static final String BROKER_B_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61617";
    private static final String BROKER_C_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61618";
    private static final String BROKER_A_NOB_TC_ADDRESS = "tcp://127.0.0.1:61626";
    private static final String BROKER_B_NOB_TC_ADDRESS = "tcp://127.0.0.1:61627";
    private static final String BROKER_C_NOB_TC_ADDRESS = "tcp://127.0.0.1:61628";
    private static final String BROKER_A_NAME = "BROKERA";
    private static final String BROKER_B_NAME = "BROKERB";
    private static final String BROKER_C_NAME = "BROKERC";

    /**
     * Basic dynamic failover 3 broker test
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterSingleConnectorBasic() throws Exception {

        initSingleTcBroker("", null, null);

        Thread.sleep(2000);

        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();
        Thread.sleep(2000);

        runTests(false, null, null, null);
    }

    /**
     * Tests a 3 broker configuration to ensure that the backup is random and
     * supported in a cluster. useExponentialBackOff is set to false and
     * maxReconnectAttempts is set to 1 to move through the list quickly for
     * this test.
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterSingleConnectorBackupFailoverConfig() throws Exception {

        initSingleTcBroker("", null, null);

        Thread.sleep(2000);

        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?backup=true&backupPoolSize=2&useExponentialBackOff=false&initialReconnectDelay=500");
        createClients();
        Thread.sleep(2000);

        runTests(false, null, null, null);
    }

    /**
     * Tests a 3 broker cluster that passes in connection params on the
     * transport connector. Prior versions of AMQ passed the TC connection
     * params to the client and this should not happen. The chosen param is not
     * compatible with the client and will throw an error if used.
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterSingleConnectorWithParams() throws Exception {

        initSingleTcBroker("?transport.closeAsync=false", null, null);

        Thread.sleep(2000);
        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();
        Thread.sleep(2000);

        runTests(false, null, null, null);
    }

    /**
     * Tests a 3 broker cluster using a cluster filter of *
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterWithClusterFilter() throws Exception {

        initSingleTcBroker("?transport.closeAsync=false", null, null);

        Thread.sleep(2000);
        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();

        runTests(false, null, "*", null);
    }

    /**
     * Test to verify that a broker with multiple transport connections only the
     * one marked to update clients is propagate
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterMultipleConnectorBasic() throws Exception {

        initMultiTcCluster("", null);

        Thread.sleep(2000);

        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();
        Thread.sleep(2000);

        runTests(true, null, null, null);
    }

    /**
     * Test to verify the reintroduction of the A Broker
     *
     * @throws Exception
     */
    public void testOriginalBrokerRestart() throws Exception {
        initSingleTcBroker("", null, null);

        Thread.sleep(2000);

        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();

        assertClientsConnectedToThreeBrokers();

        getBroker(BROKER_A_NAME).stop();
        getBroker(BROKER_A_NAME).waitUntilStopped();
        removeBroker(BROKER_A_NAME);

        assertClientsConnectedToTwoBrokers();

        createBrokerA(false, null, null, null);
        getBroker(BROKER_A_NAME).waitUntilStarted();

        assertClientsConnectedToThreeBrokers();
    }

    /**
     * Test to ensure clients are evenly to all available brokers in the
     * network.
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterClientDistributions() throws Exception {

        initSingleTcBroker("", null, null);

        Thread.sleep(2000);
        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false&initialReconnectDelay=500&randomize=false");
        createClients(100);
        Thread.sleep(5000);

        runClientDistributionTests(false, null, null, null);
    }

    /**
     * Test to verify that clients are distributed with no less than 20% of the
     * clients on any one broker.
     *
     * @throws Exception
     */
    public void testThreeBrokerClusterDestinationFilter() throws Exception {

        initSingleTcBroker("", null, null);

        Thread.sleep(2000);
        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
        createClients();

        runTests(false, null, null, "Queue.TEST.FOO.>");
    }

    public void testFailOverWithUpdateClientsOnRemove() throws Exception{
        // Broker A
        addBroker(BROKER_A_NAME, createBroker(BROKER_A_NAME));
        TransportConnector connectorA = getBroker(BROKER_A_NAME).addConnector(BROKER_A_CLIENT_TC_ADDRESS);
        connectorA.setName("openwire");
        connectorA.setRebalanceClusterClients(true);
        connectorA.setUpdateClusterClients(true);
        connectorA.setUpdateClusterClientsOnRemove(true); //If set to false the test succeeds.
        addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
        getBroker(BROKER_A_NAME).start();

        // Broker B
        addBroker(BROKER_B_NAME, createBroker(BROKER_B_NAME));
        TransportConnector connectorB = getBroker(BROKER_B_NAME).addConnector(BROKER_B_CLIENT_TC_ADDRESS);
        connectorB.setName("openwire");
        connectorB.setRebalanceClusterClients(true);
        connectorB.setUpdateClusterClients(true);
        connectorB.setUpdateClusterClientsOnRemove(true); //If set to false the test succeeds.
        addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
        getBroker(BROKER_B_NAME).start();

        getBroker(BROKER_B_NAME).waitUntilStarted();
        Thread.sleep(1000);

        // create client connecting only to A. It should receive broker B address whet it connects to A.
        setClientUrl("failover:(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=true");
        createClients(1);
        Thread.sleep(5000);

        // We stop broker A.
        logger.info("Stopping broker A whose address is: {}", BROKER_A_CLIENT_TC_ADDRESS);
        getBroker(BROKER_A_NAME).stop();
        getBroker(BROKER_A_NAME).waitUntilStopped();
        Thread.sleep(5000);

        // Client should failover to B.
        assertAllConnectedTo(BROKER_B_CLIENT_TC_ADDRESS);
    }

    public void testStaticInfoAvailableAfterPattialUpdate() throws Exception {

        addBroker(BROKER_A_NAME, createBroker(BROKER_A_NAME));
        TransportConnector connectorA = getBroker(BROKER_A_NAME).addConnector(BROKER_A_CLIENT_TC_ADDRESS);
        connectorA.setName("openwire");
        connectorA.setRebalanceClusterClients(true);
        connectorA.setUpdateClusterClients(true);
        connectorA.getPublishedAddressPolicy().setPublishedHostStrategy(PublishedAddressPolicy.PublishedHostStrategy.IPADDRESS);

        getBroker(BROKER_A_NAME).start();

        setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "?trace=true," + BROKER_B_CLIENT_TC_ADDRESS + "?trace=true)?useExponentialBackOff=false&initialReconnectDelay=500");
        createClients(1);

        assertAllConnectedTo(BROKER_A_CLIENT_TC_ADDRESS);

        getBroker(BROKER_A_NAME).stop();


        addBroker(BROKER_B_NAME, createBroker(BROKER_B_NAME));
        TransportConnector connectorB = getBroker(BROKER_B_NAME).addConnector(BROKER_B_CLIENT_TC_ADDRESS);
        connectorB.setName("openwire");
        connectorB.setRebalanceClusterClients(true);
        connectorB.setUpdateClusterClients(true);
        connectorB.getPublishedAddressPolicy().setPublishedHostStrategy(PublishedAddressPolicy.PublishedHostStrategy.IPADDRESS);

        getBroker(BROKER_B_NAME).start();

        getBroker(BROKER_B_NAME).waitUntilStarted();
        Thread.sleep(1000);

        // verify can connect?
        assertAllConnectedTo(BROKER_B_CLIENT_TC_ADDRESS);
    }

    /**
     * Runs a 3 Broker dynamic failover test: <br/>
     * <ul>
     * <li>asserts clients are distributed across all 3 brokers</li>
     * <li>asserts clients are distributed across 2 brokers after removing the 3rd</li>
     * <li>asserts clients are distributed across all 3 brokers after
     * reintroducing the 3rd broker</li>
     * </ul>
     *
     * @param multi
     * @param tcParams
     * @param clusterFilter
     * @param destinationFilter
     * @throws Exception
     * @throws InterruptedException
     */
    private void runTests(boolean multi, String tcParams, String clusterFilter, String destinationFilter) throws Exception, InterruptedException {
        assertClientsConnectedToThreeBrokers();

        LOG.info("Stopping BrokerC in prep for restart");
        getBroker(BROKER_C_NAME).stop();
        getBroker(BROKER_C_NAME).waitUntilStopped();
        removeBroker(BROKER_C_NAME);

        assertClientsConnectedToTwoBrokers();

        LOG.info("Recreating BrokerC after stop");
        createBrokerC(multi, tcParams, clusterFilter, destinationFilter);
        getBroker(BROKER_C_NAME).waitUntilStarted();

        assertClientsConnectedToThreeBrokers();
    }

    /**
     * @param multi
     * @param tcParams
     * @param clusterFilter
     * @param destinationFilter
     * @throws Exception
     * @throws InterruptedException
     */
    private void runClientDistributionTests(boolean multi, String tcParams, String clusterFilter, String destinationFilter) throws Exception, InterruptedException {
        assertClientsConnectedToThreeBrokers();
        assertClientsConnectionsEvenlyDistributed(.25);

        getBroker(BROKER_C_NAME).stop();
        getBroker(BROKER_C_NAME).waitUntilStopped();
        removeBroker(BROKER_C_NAME);

        assertClientsConnectedToTwoBrokers();
        assertClientsConnectionsEvenlyDistributed(.35);

        createBrokerC(multi, tcParams, clusterFilter, destinationFilter);
        getBroker(BROKER_C_NAME).waitUntilStarted();

        assertClientsConnectedToThreeBrokers();
        assertClientsConnectionsEvenlyDistributed(.20);
    }

    @Override
    protected void setUp() throws Exception {
    }

    @Override
    protected void tearDown() throws Exception {
        shutdownClients();
        Thread.sleep(2000);
        destroyBrokerCluster();
    }

    private void initSingleTcBroker(String params, String clusterFilter, String destinationFilter) throws Exception {
        createBrokerA(false, params, clusterFilter, null);
        createBrokerB(false, params, clusterFilter, null);
        createBrokerC(false, params, clusterFilter, null);
        getBroker(BROKER_C_NAME).waitUntilStarted();
    }

    private void initMultiTcCluster(String params, String clusterFilter) throws Exception {
        createBrokerA(true, params, clusterFilter, null);
        createBrokerB(true, params, clusterFilter, null);
        createBrokerC(true, params, clusterFilter, null);
        getBroker(BROKER_C_NAME).waitUntilStarted();
    }

    private void createBrokerA(boolean multi, String params, String clusterFilter, String destinationFilter) throws Exception {
        final String tcParams = (params == null)?"":params;
        if (getBroker(BROKER_A_NAME) == null) {
            addBroker(BROKER_A_NAME, createBroker(BROKER_A_NAME));
            addTransportConnector(getBroker(BROKER_A_NAME), "openwire", BROKER_A_CLIENT_TC_ADDRESS + tcParams, true);
            if (multi) {
                addTransportConnector(getBroker(BROKER_A_NAME), "network", BROKER_A_NOB_TC_ADDRESS + tcParams, false);
                addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_C_Bridge", "static://(" + BROKER_C_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            } else {
                addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_C_Bridge", "static://(" + BROKER_C_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            }
            getBroker(BROKER_A_NAME).start();
        }
    }

    private void createBrokerB(boolean multi, String params, String clusterFilter, String destinationFilter) throws Exception {
        final String tcParams = (params == null)?"":params;
        if (getBroker(BROKER_B_NAME) == null) {
            addBroker(BROKER_B_NAME, createBroker(BROKER_B_NAME));
            addTransportConnector(getBroker(BROKER_B_NAME), "openwire", BROKER_B_CLIENT_TC_ADDRESS + tcParams, true);
            if (multi) {
                addTransportConnector(getBroker(BROKER_B_NAME), "network", BROKER_B_NOB_TC_ADDRESS + tcParams, false);
                addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_C_Bridge", "static://(" + BROKER_C_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            } else {
                addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_C_Bridge", "static://(" + BROKER_C_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            }
            getBroker(BROKER_B_NAME).start();
        }
    }

    private void createBrokerC(boolean multi, String params, String clusterFilter, String destinationFilter) throws Exception {
        final String tcParams = (params == null)?"":params;
        if (getBroker(BROKER_C_NAME) == null) {
            addBroker(BROKER_C_NAME, createBroker(BROKER_C_NAME));
            addTransportConnector(getBroker(BROKER_C_NAME), "openwire", BROKER_C_CLIENT_TC_ADDRESS + tcParams, true);
            if (multi) {
                addTransportConnector(getBroker(BROKER_C_NAME), "network", BROKER_C_NOB_TC_ADDRESS + tcParams, false);
                addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_A_Bridge", "static://(" + BROKER_A_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_B_Bridge", "static://(" + BROKER_B_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            } else {
                addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
                addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
            }
            getBroker(BROKER_C_NAME).start();
        }
    }
}
