content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  
  package com.cloud.network.router;
  
  import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
  import static com.cloud.vm.VirtualMachineManager.SystemVmEnableUserData;
  
  import java.lang.reflect.Type;
  import java.math.BigInteger;
  import java.nio.charset.Charset;
  import java.security.MessageDigest;
  import java.security.NoSuchAlgorithmException;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Calendar;
  import java.util.Collections;
  import java.util.Date;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.TimeZone;
  import java.util.concurrent.BlockingQueue;
  import java.util.concurrent.ExecutorService;
  import java.util.concurrent.Executors;
  import java.util.concurrent.LinkedBlockingQueue;
  import java.util.concurrent.ScheduledExecutorService;
  import java.util.concurrent.TimeUnit;
  import java.util.regex.Pattern;
  
  import javax.inject.Inject;
  import javax.naming.ConfigurationException;
  
  import com.google.gson.JsonSyntaxException;
  import com.google.gson.reflect.TypeToken;
  
  import org.apache.cloudstack.alert.AlertService;
  import org.apache.cloudstack.alert.AlertService.AlertType;
  import org.apache.cloudstack.api.ApiCommandResourceType;
  import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
  import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
  import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
  import org.apache.cloudstack.config.ApiServiceConfiguration;
  import org.apache.cloudstack.context.CallContext;
  import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
  import org.apache.cloudstack.framework.config.ConfigKey;
  import org.apache.cloudstack.framework.config.Configurable;
  import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
  import org.apache.cloudstack.framework.jobs.AsyncJobManager;
  import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
  import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO;
  import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao;
  import org.apache.cloudstack.managed.context.ManagedContextRunnable;
  import org.apache.cloudstack.network.BgpPeer;
  import org.apache.cloudstack.network.RoutedIpv4Manager;
  import org.apache.cloudstack.network.topology.NetworkTopology;
  import org.apache.cloudstack.network.topology.NetworkTopologyContext;
  import org.apache.cloudstack.userdata.UserDataManager;
  import org.apache.cloudstack.utils.CloudStackVersion;
  import org.apache.cloudstack.utils.identity.ManagementServerNode;
  import org.apache.cloudstack.utils.usage.UsageUtils;
  import org.apache.commons.collections4.CollectionUtils;
  import org.apache.commons.lang3.ObjectUtils;
  import org.apache.commons.lang3.StringUtils;
  import org.springframework.beans.factory.annotation.Autowired;
  import org.springframework.beans.factory.annotation.Qualifier;
  
  import com.cloud.agent.AgentManager;
  import com.cloud.agent.Listener;
  import com.cloud.agent.api.AgentControlAnswer;
  import com.cloud.agent.api.AgentControlCommand;
  import com.cloud.agent.api.Answer;
  import com.cloud.agent.api.CheckRouterAnswer;
  import com.cloud.agent.api.CheckRouterCommand;
  import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer;
  import com.cloud.agent.api.CheckS2SVpnConnectionsCommand;
  import com.cloud.agent.api.Command;
  import com.cloud.agent.api.GetDomRVersionAnswer;
  import com.cloud.agent.api.GetDomRVersionCmd;
  import com.cloud.agent.api.GetRouterAlertsAnswer;
  import com.cloud.agent.api.NetworkUsageAnswer;
  import com.cloud.agent.api.NetworkUsageCommand;
  import com.cloud.agent.api.StartupCommand;
  import com.cloud.agent.api.check.CheckSshCommand;
  import com.cloud.agent.api.routing.AggregationControlCommand;
  import com.cloud.agent.api.routing.AggregationControlCommand.Action;
  import com.cloud.agent.api.routing.GetRouterAlertsCommand;
  import com.cloud.agent.api.routing.GetRouterMonitorResultsAnswer;
  import com.cloud.agent.api.routing.GetRouterMonitorResultsCommand;
  import com.cloud.agent.api.routing.GroupAnswer;
  import com.cloud.agent.api.routing.IpAliasTO;
  import com.cloud.agent.api.routing.NetworkElementCommand;
  import com.cloud.agent.api.routing.SetMonitorServiceCommand;
  import com.cloud.agent.api.to.MonitorServiceTO;
  import com.cloud.agent.manager.Commands;
  import com.cloud.alert.AlertManager;
  import com.cloud.api.ApiAsyncJobDispatcher;
  import com.cloud.api.ApiGsonHelper;
  import com.cloud.api.query.dao.DomainRouterJoinDao;
  import com.cloud.api.query.dao.UserVmJoinDao;
  import com.cloud.api.query.vo.DomainRouterJoinVO;
  import com.cloud.api.query.vo.UserVmJoinVO;
  import com.cloud.bgp.BGPService;
  import com.cloud.cluster.ManagementServerHostVO;
  import com.cloud.cluster.dao.ManagementServerHostDao;
  import com.cloud.configuration.Config;
  import com.cloud.configuration.ZoneConfig;
  import com.cloud.dc.DataCenter;
  import com.cloud.dc.DataCenter.NetworkType;
  import com.cloud.dc.DataCenterVO;
  import com.cloud.dc.HostPodVO;
  import com.cloud.dc.dao.DataCenterDao;
  import com.cloud.dc.dao.HostPodDao;
  import com.cloud.dc.dao.VlanDao;
  import com.cloud.deploy.DeployDestination;
  import com.cloud.deploy.DeploymentPlanner;
  import com.cloud.domain.Domain;
  import com.cloud.event.ActionEvent;
  import com.cloud.event.ActionEventUtils;
  import com.cloud.event.EventTypes;
  import com.cloud.exception.AgentUnavailableException;
  import com.cloud.exception.ConcurrentOperationException;
  import com.cloud.exception.ConnectionException;
  import com.cloud.exception.InsufficientCapacityException;
  import com.cloud.exception.InvalidParameterValueException;
  import com.cloud.exception.OperationTimedoutException;
  import com.cloud.exception.ResourceUnavailableException;
  import com.cloud.host.Host;
  import com.cloud.host.HostVO;
  import com.cloud.host.Status;
  import com.cloud.host.dao.HostDao;
  import com.cloud.hypervisor.Hypervisor.HypervisorType;
  import com.cloud.network.IpAddress;
  import com.cloud.network.MonitoringService;
  import com.cloud.network.Network;
  import com.cloud.network.Network.GuestType;
  import com.cloud.network.Network.Provider;
  import com.cloud.network.Network.Service;
  import com.cloud.network.NetworkModel;
  import com.cloud.network.NetworkService;
  import com.cloud.network.Networks.TrafficType;
  import com.cloud.network.PublicIpAddress;
  import com.cloud.network.RemoteAccessVpn;
  import com.cloud.network.RouterHealthCheckResult;
  import com.cloud.network.Site2SiteCustomerGateway;
  import com.cloud.network.Site2SiteVpnConnection;
  import com.cloud.network.SshKeysDistriMonitor;
  import com.cloud.network.VirtualNetworkApplianceService;
  import com.cloud.network.VirtualRouterProvider;
  import com.cloud.network.addr.PublicIp;
  import com.cloud.network.dao.FirewallRulesDao;
  import com.cloud.network.dao.IPAddressDao;
  import com.cloud.network.dao.IPAddressVO;
  import com.cloud.network.dao.LBStickinessPolicyDao;
  import com.cloud.network.dao.LBStickinessPolicyVO;
  import com.cloud.network.dao.LoadBalancerDao;
  import com.cloud.network.dao.LoadBalancerVMMapDao;
  import com.cloud.network.dao.LoadBalancerVMMapVO;
  import com.cloud.network.dao.LoadBalancerVO;
  import com.cloud.network.dao.MonitoringServiceDao;
  import com.cloud.network.dao.MonitoringServiceVO;
  import com.cloud.network.dao.NetworkDao;
  import com.cloud.network.dao.NetworkServiceMapDao;
  import com.cloud.network.dao.NetworkVO;
  import com.cloud.network.dao.OpRouterMonitorServiceDao;
  import com.cloud.network.dao.OpRouterMonitorServiceVO;
  import com.cloud.network.dao.RemoteAccessVpnDao;
  import com.cloud.network.dao.RouterHealthCheckResultDao;
  import com.cloud.network.dao.RouterHealthCheckResultVO;
  import com.cloud.network.dao.Site2SiteCustomerGatewayDao;
  import com.cloud.network.dao.Site2SiteVpnConnectionDao;
  import com.cloud.network.dao.Site2SiteVpnConnectionVO;
  import com.cloud.network.dao.VirtualRouterProviderDao;
  import com.cloud.network.lb.LoadBalancingRule;
  import com.cloud.network.lb.LoadBalancingRule.LbDestination;
  import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy;
  import com.cloud.network.lb.LoadBalancingRule.LbSslCert;
  import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy;
  import com.cloud.network.lb.LoadBalancingRulesManager;
  import com.cloud.network.router.VirtualRouter.RedundantState;
  import com.cloud.network.router.VirtualRouter.Role;
  import com.cloud.network.rules.FirewallRule;
  import com.cloud.network.rules.FirewallRule.Purpose;
  import com.cloud.network.rules.FirewallRuleVO;
  import com.cloud.network.rules.LoadBalancerContainer.Scheme;
  import com.cloud.network.rules.PortForwardingRule;
  import com.cloud.network.rules.PortForwardingRuleVO;
  import com.cloud.network.rules.RulesManager;
  import com.cloud.network.rules.StaticNat;
  import com.cloud.network.rules.StaticNatImpl;
  import com.cloud.network.rules.StaticNatRule;
  import com.cloud.network.rules.dao.PortForwardingRulesDao;
  import com.cloud.network.vpc.Vpc;
  import com.cloud.network.vpc.VpcManager;
  import com.cloud.network.vpc.VpcService;
  import com.cloud.network.vpc.dao.VpcDao;
  import com.cloud.network.vpn.Site2SiteVpnManager;
  import com.cloud.offering.DiskOffering;
  import com.cloud.offering.NetworkOffering;
  import com.cloud.offering.ServiceOffering;
  import com.cloud.offerings.NetworkOfferingVO;
  import com.cloud.offerings.dao.NetworkOfferingDao;
  import com.cloud.serializer.GsonHelper;
  import com.cloud.server.ManagementServer;
  import com.cloud.service.ServiceOfferingVO;
  import com.cloud.service.dao.ServiceOfferingDao;
  import com.cloud.storage.Storage.ProvisioningType;
  import com.cloud.user.Account;
  import com.cloud.user.AccountManager;
  import com.cloud.user.User;
  import com.cloud.user.UserStatisticsVO;
  import com.cloud.user.UserStatsLogVO;
  import com.cloud.user.UserVO;
  import com.cloud.user.dao.UserDao;
  import com.cloud.user.dao.UserStatisticsDao;
  import com.cloud.user.dao.UserStatsLogDao;
  import com.cloud.utils.NumbersUtil;
  import com.cloud.utils.Pair;
  import com.cloud.utils.component.ComponentContext;
  import com.cloud.utils.component.ManagerBase;
  import com.cloud.utils.concurrency.NamedThreadFactory;
  import com.cloud.utils.db.DB;
  import com.cloud.utils.db.EntityManager;
  import com.cloud.utils.db.Filter;
  import com.cloud.utils.db.GlobalLock;
  import com.cloud.utils.db.QueryBuilder;
  import com.cloud.utils.db.SearchBuilder;
  import com.cloud.utils.db.SearchCriteria;
  import com.cloud.utils.db.Transaction;
  import com.cloud.utils.db.TransactionCallbackNoReturn;
  import com.cloud.utils.db.TransactionStatus;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.fsm.StateListener;
  import com.cloud.utils.fsm.StateMachine2;
  import com.cloud.utils.net.Ip;
  import com.cloud.utils.net.MacAddress;
  import com.cloud.utils.net.NetUtils;
  import com.cloud.vm.DomainRouterVO;
  import com.cloud.vm.Nic;
  import com.cloud.vm.NicIpAlias;
  import com.cloud.vm.NicProfile;
  import com.cloud.vm.NicVO;
  import com.cloud.vm.ReservationContext;
  import com.cloud.vm.ReservationContextImpl;
  import com.cloud.vm.VirtualMachine;
  import com.cloud.vm.VirtualMachineGuru;
  import com.cloud.vm.VirtualMachineManager;
  import com.cloud.vm.VirtualMachineProfile;
  import com.cloud.vm.VirtualMachineProfile.Param;
  import com.cloud.vm.dao.DomainRouterDao;
  import com.cloud.vm.dao.NicDao;
  import com.cloud.vm.dao.NicIpAliasDao;
  import com.cloud.vm.dao.NicIpAliasVO;
  import com.cloud.vm.dao.VMInstanceDetailsDao;
  
  /**
   * VirtualNetworkApplianceManagerImpl manages the different types of virtual
   * network appliances available in the Cloud Stack.
   */
  public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, VirtualMachineGuru, Listener,
  Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualMachine> {
      private static final String CONNECTIVITY_TEST = "connectivity.test";
      private static final String FILESYSTEM_WRITABLE_TEST = "filesystem.writable.test";
      private static final String READONLY_FILESYSTEM_ERROR = "Read-only file system";
      private static final String BACKUP_ROUTER_EXCLUDED_TESTS = "gateways_check.py";
      /**
       * Used regex to ensure that the value that will be passed to the VR is an acceptable value
       */
      public static final String loggerROTATE_REGEX = "((?i)(hourly)|(daily)|(monthly))|(\\*|\\d{2})\\:(\\*|\\d{2})\\:(\\*|\\d{2})";
  
      @Inject EntityManager _entityMgr;
      @Inject DataCenterDao _dcDao;
      @Inject VlanDao _vlanDao;
      @Inject FirewallRulesDao _rulesDao;
      @Inject LoadBalancerDao _loadBalancerDao;
      @Inject LoadBalancerVMMapDao _loadBalancerVMMapDao;
      @Inject protected IPAddressDao _ipAddressDao;
      @Inject protected DomainRouterDao _routerDao;
      @Inject UserDao _userDao;
      @Inject protected UserStatisticsDao _userStatsDao;
      @Inject HostDao _hostDao;
      @Inject ConfigurationDao _configDao;
      @Inject HostPodDao _podDao;
      @Inject UserStatsLogDao _userStatsLogDao;
      @Inject protected AgentManager _agentMgr;
      @Inject AlertManager _alertMgr;
      @Inject AccountManager _accountMgr;
      @Inject protected ServiceOfferingDao _serviceOfferingDao;
      @Inject NetworkOfferingDao _networkOfferingDao;
      @Inject protected NetworkOrchestrationService _networkMgr;
      @Inject protected NetworkModel _networkModel;
      @Inject protected VirtualMachineManager _itMgr;
      @Inject RulesManager _rulesMgr;
      @Inject protected NetworkDao _networkDao;
      @Inject LoadBalancingRulesManager _lbMgr;
      @Inject PortForwardingRulesDao _pfRulesDao;
      @Inject protected RemoteAccessVpnDao _vpnDao;
      @Inject protected NicDao _nicDao;
      @Inject NicIpAliasDao _nicIpAliasDao;
      @Inject VMInstanceDetailsDao _vmDetailsDao;
      @Inject protected VirtualRouterProviderDao _vrProviderDao;
      @Inject ManagementServerHostDao _msHostDao;
      @Inject Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao;
      @Inject Site2SiteVpnConnectionDao _s2sVpnConnectionDao;
      @Inject Site2SiteVpnManager _s2sVpnMgr;
      @Inject NetworkService _networkSvc;
      @Inject protected MonitoringServiceDao _monitorServiceDao;
      @Inject AsyncJobManager _asyncMgr;
      @Inject protected VpcDao _vpcDao;
      @Inject protected ApiAsyncJobDispatcher _asyncDispatcher;
      @Inject OpRouterMonitorServiceDao _opRouterMonitorServiceDao;
  
      @Inject protected NetworkTopologyContext _networkTopologyContext;
  
      @Inject UserVmJoinDao userVmJoinDao;
      @Inject DomainRouterJoinDao domainRouterJoinDao;
      @Inject PortForwardingRulesDao portForwardingDao;
      @Inject ApplicationLoadBalancerRuleDao applicationLoadBalancerRuleDao;
      @Inject RouterHealthCheckResultDao routerHealthCheckResultDao;
      @Inject LBStickinessPolicyDao lbStickinessPolicyDao;
      @Inject NetworkServiceMapDao _ntwkSrvcDao;
  
      @Inject NetworkService networkService;
      @Inject VpcService vpcService;
      @Inject private VpcManager vpcManager;
  
      @Autowired
      @Qualifier("networkHelper")
      protected NetworkHelper _nwHelper;
  
      @Inject protected RouterControlHelper _routerControlHelper;
  
      @Inject protected CommandSetupHelper _commandSetupHelper;
      @Inject ManagementServer mgr;
      @Inject
      RoutedIpv4Manager routedIpv4Manager;
      @Inject
      BGPService bgpService;
  
      @Inject
      private UserDataManager userDataManager;
      private int _routerStatsInterval = 300;
      private int _routerCheckInterval = 30;
      private int _rvrStatusUpdatePoolSize = 10;
      private String _dnsBasicZoneUpdates = "all";
  
      private boolean _disableRpFilter = false;
      private int _routerExtraPublicNics = 2;
      private int _usageAggregationRange = 1440;
      private String _usageTimeZone = "GMT";
      private final long mgmtSrvrId = MacAddress.getMacAddress().toLong();
      private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds
      private boolean _dailyOrHourly = false;
  
      private ScheduledExecutorService _executor;
      private ScheduledExecutorService _checkExecutor;
      private ScheduledExecutorService _networkStatsUpdateExecutor;
      private ExecutorService _rvrStatusUpdateExecutor;
  
      private BlockingQueue<Long> _vrUpdateQueue;
  
      @Override
      public VirtualRouter destroyRouter(final long routerId, final Account caller, final Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException {
          return _nwHelper.destroyRouter(routerId, caller, callerUserId);
      }
  
      @Override
      @DB
      public VirtualRouter upgradeRouter(final UpgradeRouterCmd cmd) {
          final Long routerId = cmd.getId();
          final Long serviceOfferingId = cmd.getServiceOfferingId();
          final Account caller = CallContext.current().getCallingAccount();
  
          final DomainRouterVO router = _routerDao.findById(routerId);
          if (router == null) {
              throw new InvalidParameterValueException("Unable to find router with id " + routerId);
          }
  
          _accountMgr.checkAccess(caller, null, true, router);
  
          if (router.getServiceOfferingId() == serviceOfferingId) {
              logger.debug("Router: {} already has service offering: {}", router, serviceOfferingId);
              return _routerDao.findById(routerId);
          }
  
          final ServiceOffering newServiceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId);
          if (newServiceOffering == null) {
              throw new InvalidParameterValueException("Unable to find service offering with id " + serviceOfferingId);
          }
          DiskOffering newDiskOffering = _entityMgr.findById(DiskOffering.class, newServiceOffering.getDiskOfferingId());
          if (newDiskOffering == null) {
              throw new InvalidParameterValueException("Unable to find disk offering: " + newServiceOffering.getDiskOfferingId());
          }
  
          // check if it is a system service offering, if yes return with error as
          // it cannot be used for user vms
          if (!newServiceOffering.isSystemUse()) {
              throw new InvalidParameterValueException(String.format("Cannot upgrade router vm to a non system service offering %s", newServiceOffering));
          }
  
          // Check that the router is stopped
          if (!router.getState().equals(VirtualMachine.State.Stopped)) {
              logger.warn("Unable to upgrade router " + router + " in state " + router.getState());
              throw new InvalidParameterValueException("Unable to upgrade router " + router + " in state " + router.getState()
                      + "; make sure the router is stopped and not in an error state before upgrading.");
          }
  
          // Check that the service offering being upgraded to has the same
          // storage pool preference as the VM's current service
          // offering
          if (_itMgr.isRootVolumeOnLocalStorage(routerId) != newDiskOffering.isUseLocalStorage()) {
              throw new InvalidParameterValueException(String.format(
                      "Can't upgrade, due to new local storage status : %s is different from current local storage status of router %s",
                      newDiskOffering.isUseLocalStorage(), router));
          }
  
          router.setServiceOfferingId(serviceOfferingId);
          if (_routerDao.update(routerId, router)) {
              return _routerDao.findById(routerId);
          } else {
              throw new CloudRuntimeException("Unable to upgrade router " + router);
          }
  
      }
  
      @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true)
      @Override
      public VirtualRouter stopRouter(final long routerId, final boolean forced) throws ResourceUnavailableException, ConcurrentOperationException {
          final CallContext context = CallContext.current();
          final Account account = context.getCallingAccount();
  
          // verify parameters
          final DomainRouterVO router = _routerDao.findById(routerId);
          if (router == null) {
              throw new InvalidParameterValueException("Unable to find router by id " + routerId + ".");
          }
  
          _accountMgr.checkAccess(account, null, true, router);
  
          final UserVO user = _userDao.findById(CallContext.current().getCallingUserId());
  
          final VirtualRouter virtualRouter = stop(router, forced, user, account);
          if (virtualRouter == null) {
              throw new CloudRuntimeException("Failed to stop router " + router);
          }
  
          // Clear stop pending flag after stopped successfully
          if (router.isStopPending()) {
              logger.info("Clear the stop pending flag of router " + router.getHostName() + " after stop router successfully");
              router.setStopPending(false);
              _routerDao.persist(router);
              virtualRouter.setStopPending(false);
          }
          return virtualRouter;
      }
  
      @DB
      public void processStopOrRebootAnswer(final DomainRouterVO router, final Answer ignoredAnswer) {
          Transaction.execute(new TransactionCallbackNoReturn() {
              @Override
              public void doInTransactionWithoutResult(final TransactionStatus status) {
                  // FIXME!!! - UserStats command should grab bytesSent/Received
                  // for all guest interfaces of the VR
                  final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
                  for (final Long guestNtwkId : routerGuestNtwkIds) {
                      final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), guestNtwkId, null, router.getId(), router.getType()
                              .toString());
                      if (userStats != null) {
                          final long currentBytesRcvd = userStats.getCurrentBytesReceived();
                          userStats.setCurrentBytesReceived(0);
                          userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd);
  
                          final long currentBytesSent = userStats.getCurrentBytesSent();
                          userStats.setCurrentBytesSent(0);
                          userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent);
                          _userStatsDao.update(userStats.getId(), userStats);
                          logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop");
                      } else {
                          DataCenterVO zone = _dcDao.findById(router.getDataCenterId());
                          Account account = _accountMgr.getAccount(router.getAccountId());
                          logger.warn("User stats for router {} were not created for account {} and dc {}", router, account, zone);
                      }
                  }
              }
          });
      }
  
      @Override
      @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true)
      public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork, final boolean forced) throws ConcurrentOperationException, ResourceUnavailableException,
      InsufficientCapacityException {
          final Account caller = CallContext.current().getCallingAccount();
  
          // verify parameters
          final DomainRouterVO router = _routerDao.findById(routerId);
          if (router == null) {
              throw new InvalidParameterValueException("Unable to find domain router with id " + routerId + ".");
          }
  
          _accountMgr.checkAccess(caller, null, true, router);
  
          // Can reboot domain router only in Running state
          if (router.getState() != VirtualMachine.State.Running) {
              logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState());
              throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), DataCenter.class, router.getDataCenterId());
          }
  
          final UserVO user = _userDao.findById(CallContext.current().getCallingUserId());
          logger.debug("Stopping and starting router " + router + " as a part of router reboot");
  
          if (stop(router, forced, user, caller) != null) {
              return startRouter(routerId, reprogramNetwork);
          } else {
              throw new CloudRuntimeException("Failed to reboot router " + router);
          }
      }
  
      @Override
      public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
  
          _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor"));
          _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor"));
          _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater"));
  
          VirtualMachine.State.getStateMachine().registerListener(this);
  
          final Map<String, String> configs = _configDao.getConfiguration("AgentManager", params);
  
          int routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE);
          int routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ);
  
          _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2);
  
          String value = configs.get("router.stats.interval");
          _routerStatsInterval = NumbersUtil.parseInt(value, 300);
  
          value = configs.get("router.check.interval");
          _routerCheckInterval = NumbersUtil.parseInt(value, 30);
  
          value = configs.get("router.check.poolsize");
          _rvrStatusUpdatePoolSize = NumbersUtil.parseInt(value, 10);
  
          /*
           * We assume that one thread can handle 20 requests in 1 minute in
           * normal situation, so here we give the queue size up to 50 minutes.
           * It's mostly for buffer, since each time CheckRouterTask running, it
           * would add all the redundant networks in the queue immediately
           */
          _vrUpdateQueue = new LinkedBlockingQueue<>(_rvrStatusUpdatePoolSize * 1000);
  
          _rvrStatusUpdateExecutor = Executors.newFixedThreadPool(_rvrStatusUpdatePoolSize, new NamedThreadFactory("RedundantRouterStatusMonitor"));
  
          String instance = configs.get("instance.name");
          if (instance == null) {
              instance = "DEFAULT";
          }
  
          NetworkHelperImpl.setVMInstanceName(instance);
  
          final String rpValue = configs.get("network.disable.rpfilter");
          if (rpValue != null && rpValue.equalsIgnoreCase("true")) {
              _disableRpFilter = true;
          }
  
          _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key()));
  
          logger.info("Router configurations: " + "ramsize=" + routerRamSize);
  
          _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false);
  
          final List<ServiceOfferingVO> offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Software Router",
                  ServiceOffering.routerDefaultOffUniqueName, 1, routerRamSize, routerCpuMHz, null,
                  null, true, null, ProvisioningType.THIN, true, null, true, VirtualMachine.Type.DomainRouter, true);
          // this can sometimes happen, if DB is manually or programmatically manipulated
          if (offerings == null || offerings.size() < 2) {
              final String msg = "Data integrity problem : System Offering For Software router VM has been removed?";
              logger.error(msg);
              throw new ConfigurationException(msg);
          }
  
          NetworkHelperImpl.setSystemAccount(_accountMgr.getSystemAccount());
  
          final String aggregationRange = configs.get("usage.stats.job.aggregation.range");
          _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440);
          _usageTimeZone = configs.get("usage.aggregation.timezone");
          if (_usageTimeZone == null) {
              _usageTimeZone = "GMT";
          }
  
          _agentMgr.registerForHostEvents(this, true, false, false);
  
          logger.info("DomainRouterManager is configured.");
  
          return true;
      }
  
      @Override
      public boolean start() {
          if (_routerStatsInterval > 0) {
              _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS);
          } else {
              logger.debug("router.stats.interval - " + _routerStatsInterval + " so not scheduling the router stats thread");
          }
  
          //Schedule Network stats update task
          //Network stats aggregation should align with aggregation range
          //For daily aggregation, update stats at the end of the day
          //For hourly aggregation, update stats at the end of the hour
          final TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone);
          final Calendar cal = Calendar.getInstance(usageTimezone);
          cal.setTime(new Date());
          //aggDate is the time in millis when the aggregation should happen
          long aggDate;
          final int HOURLY_TIME = 60;
          final int DAILY_TIME = 60 * 24;
          if (_usageAggregationRange == DAILY_TIME) {
              cal.set(Calendar.HOUR_OF_DAY, 0);
              cal.set(Calendar.MINUTE, 0);
              cal.set(Calendar.SECOND, 0);
              cal.set(Calendar.MILLISECOND, 0);
              cal.roll(Calendar.DAY_OF_YEAR, true);
              cal.add(Calendar.MILLISECOND, -1);
              aggDate = cal.getTime().getTime();
              _dailyOrHourly = true;
          } else if (_usageAggregationRange == HOURLY_TIME) {
              cal.set(Calendar.MINUTE, 0);
              cal.set(Calendar.SECOND, 0);
              cal.set(Calendar.MILLISECOND, 0);
              cal.roll(Calendar.HOUR_OF_DAY, true);
              cal.add(Calendar.MILLISECOND, -1);
              aggDate = cal.getTime().getTime();
              _dailyOrHourly = true;
          } else {
              aggDate = cal.getTime().getTime();
              _dailyOrHourly = false;
          }
  
          if (_usageAggregationRange < UsageUtils.USAGE_AGGREGATION_RANGE_MIN) {
              logger.warn("Usage stats job aggregation range is to small, using the minimum value of " + UsageUtils.USAGE_AGGREGATION_RANGE_MIN);
              _usageAggregationRange = UsageUtils.USAGE_AGGREGATION_RANGE_MIN;
          }
  
          // We cannot schedule a job at specific time. Provide initial delay instead, from current time, so that the job runs at desired time
          final long initialDelay = aggDate - System.currentTimeMillis();
  
          if( initialDelay < 0){
              logger.warn("Initial delay for network usage stats update task is incorrect. Stats update task will run immediately");
          }
  
          _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), initialDelay, (long) _usageAggregationRange * 60 * 1000,
                  TimeUnit.MILLISECONDS);
  
          if (_routerCheckInterval > 0) {
              _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS);
              for (int i = 0; i < _rvrStatusUpdatePoolSize; i++) {
                  _rvrStatusUpdateExecutor.execute(new RvRStatusUpdateTask());
              }
          } else {
              logger.debug("router.check.interval - " + _routerCheckInterval + " so not scheduling the redundant router checking thread");
          }
  
          final int routerAlertsCheckInterval = RouterAlertsCheckInterval.value();
          if (routerAlertsCheckInterval > 0) {
              _checkExecutor.scheduleAtFixedRate(new CheckRouterAlertsTask(), routerAlertsCheckInterval, routerAlertsCheckInterval, TimeUnit.SECONDS);
          } else {
              logger.debug(RouterAlertsCheckIntervalCK + "=" + routerAlertsCheckInterval + " so not scheduling the router alerts checking thread");
          }
  
          final int routerHealthCheckConfigRefreshInterval = RouterHealthChecksConfigRefreshInterval.value();
          if (routerHealthCheckConfigRefreshInterval > 0) {
              _checkExecutor.scheduleAtFixedRate(new UpdateRouterHealthChecksConfigTask(), routerHealthCheckConfigRefreshInterval, routerHealthCheckConfigRefreshInterval, TimeUnit.MINUTES);
          } else {
              logger.debug(RouterHealthChecksConfigRefreshIntervalCK + "=" + routerHealthCheckConfigRefreshInterval + " so not scheduling the router health check data thread");
          }
  
          final int routerHealthChecksFetchInterval = RouterHealthChecksResultFetchInterval.value();
          if (routerHealthChecksFetchInterval > 0) {
              _checkExecutor.scheduleAtFixedRate(new FetchRouterHealthChecksResultTask(), routerHealthChecksFetchInterval, routerHealthChecksFetchInterval, TimeUnit.MINUTES);
          } else {
              logger.debug(RouterHealthChecksResultFetchIntervalCK + "=" + routerHealthChecksFetchInterval + " so not scheduling the router checks fetching thread");
          }
  
          return true;
      }
  
      @Override
      public boolean stop() {
          return true;
      }
  
      protected VirtualNetworkApplianceManagerImpl() {
      }
  
      protected class NetworkUsageTask extends ManagedContextRunnable {
  
          public NetworkUsageTask() {
          }
  
          @Override
          protected void runInContext() {
              try {
                  final List<DomainRouterVO> routers = _routerDao.listByStateAndNetworkType(VirtualMachine.State.Running, GuestType.Isolated, mgmtSrvrId);
                  logger.debug("Found {} running routers. ", routers.size());
  
                  for (final DomainRouterVO router : routers) {
                      collectNetworkStatistics(router, null);
                  }
              } catch (final Exception e) {
                  logger.warn("Error while collecting network stats", e);
              }
          }
      }
  
      protected class NetworkStatsUpdateTask extends ManagedContextRunnable {
  
          public NetworkStatsUpdateTask() {
          }
  
          @Override
          protected void runInContext() {
              final GlobalLock scanLock = GlobalLock.getInternLock("network.stats");
              try {
                  if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) {
                      // Check for ownership
                      // msHost in UP state with min id should run the job
                      final ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", false, 0L, 1L));
                      if (msHost == null || msHost.getMsid() != mgmtSrvrId) {
                          logger.debug("Skipping aggregate network stats update");
                          scanLock.unlock();
                          return;
                      }
                      try {
                          Transaction.execute(new TransactionCallbackNoReturn() {
                              @Override
                              public void doInTransactionWithoutResult(final TransactionStatus status) {
                                  // get all stats with delta > 0
                                  final List<UserStatisticsVO> updatedStats = _userStatsDao.listUpdatedStats();
                                  final Date updatedTime = new Date();
                                  for (final UserStatisticsVO stat : updatedStats) {
                                      // update agg bytes
                                      stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived());
                                      stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent());
                                      _userStatsDao.update(stat.getId(), stat);
                                      // insert into op_user_stats_log
                                      final UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat
                                              .getCurrentBytesReceived(), stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime);
                                      _userStatsLogDao.persist(statsLog);
                                  }
                                  logger.debug("Successfully updated aggregate network stats");
                              }
                          });
                      } catch (final Exception e) {
                          logger.debug("Failed to update aggregate network stats", e);
                      } finally {
                          scanLock.unlock();
                      }
                  }
              } catch (final Exception e) {
                  logger.debug("Exception while trying to acquire network stats lock", e);
              } finally {
                  scanLock.releaseRef();
              }
          }
      }
  
      @DB
      protected void updateSite2SiteVpnConnectionState(final List<DomainRouterVO> routers) {
          for (final DomainRouterVO router : routers) {
              if (router.getRole() == Role.INTERNAL_LB_VM) {
                  continue;
              }
  
              final List<Site2SiteVpnConnectionVO> conns = _s2sVpnMgr.getConnectionsForRouter(router);
              if (conns == null || conns.isEmpty()) {
                  continue;
              }
              if (router.getIsRedundantRouter() && router.getRedundantState() != RedundantState.PRIMARY){
                  continue;
              }
              if (router.getState() != VirtualMachine.State.Running) {
                  for (final Site2SiteVpnConnectionVO conn : conns) {
                      if (conn.getState() != Site2SiteVpnConnection.State.Error) {
                          conn.setState(Site2SiteVpnConnection.State.Disconnected);
                          _s2sVpnConnectionDao.persist(conn);
                      }
                  }
                  continue;
              }
              final List<String> ipList = new ArrayList<>();
              for (final Site2SiteVpnConnectionVO conn : conns) {
                  if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected
                      && conn.getState() != Site2SiteVpnConnection.State.Connecting) {
                      continue;
                  }
                  final Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId());
                  ipList.add(gw.getGatewayIp());
              }
              final String privateIP = router.getPrivateIpAddress();
              final HostVO host = _hostDao.findById(router.getHostId());
              boolean hostAvailable =  !(host == null || host.getState() != Status.Up)
                      && (host.getManagementServerId() == ManagementServerNode.getManagementServerId())
                      && (privateIP != null);
              if (hostAvailable) {
                  final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList);
                  command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
                  command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
                  command.setWait(30);
                  final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
                  CheckS2SVpnConnectionsAnswer answer;
                  if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) {
                      answer = (CheckS2SVpnConnectionsAnswer) origAnswer;
                  } else {
                      logger.warn("Unable to update router {}'s VPN connection status", router.getHostName());
                      continue;
                  }
                  if (!answer.getResult()) {
                      logger.warn("Unable to update router {}'s VPN connection status", router.getHostName());
                      continue;
                  }
                  for (final Site2SiteVpnConnectionVO conn : conns) {
                      final Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId());
                      if (lock == null) {
                          throw new CloudRuntimeException(String.format("Unable to acquire lock for site to site vpn connection %s", conn));
                      }
                      try {
                          if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected && conn.getState() != Site2SiteVpnConnection.State.Connecting) {
                              continue;
                          }
                          final Site2SiteVpnConnection.State oldState = conn.getState();
                          final Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId());
  
                          if (answer.isIPPresent(gw.getGatewayIp())) {
                              if (answer.isConnected(gw.getGatewayIp())) {
                                  conn.setState(Site2SiteVpnConnection.State.Connected);
                              } else {
                                  conn.setState(Site2SiteVpnConnection.State.Disconnected);
                              }
                              _s2sVpnConnectionDao.persist(conn);
                              if (oldState != conn.getState()) {
                                  final String title = "Site-to-site Vpn Connection to " + gw.getName() + " just switched from " + oldState + " to " + conn.getState();
                                  final String context = String.format(
                                          "Site-to-site Vpn Connection to %s on router %s(%s)  just switched from %s to %s",
                                          gw.getName(), router.getHostName(), router, oldState, conn.getState());
                                  logger.info(context);
                                  _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
                              }
                          }
                      } finally {
                          _s2sVpnConnectionDao.releaseFromLockTable(lock.getId());
                      }
                  }
              }
          }
      }
  
      protected void updateRoutersRedundantState(final List<DomainRouterVO> routers) {
          boolean updated;
          for (final DomainRouterVO router : routers) {
              updated = false;
              if (!router.getIsRedundantRouter()) {
                  continue;
              }
              final RedundantState prevState = router.getRedundantState();
              if (router.getState() != VirtualMachine.State.Running) {
                  router.setRedundantState(RedundantState.UNKNOWN);
                  updated = true;
              } else {
                  final String privateIP = router.getPrivateIpAddress();
                  final HostVO host = _hostDao.findById(router.getHostId());
                  if (host == null || host.getState() != Status.Up) {
                      router.setRedundantState(RedundantState.UNKNOWN);
                      updated = true;
                  } else if (privateIP != null) {
                      final CheckRouterCommand command = new CheckRouterCommand();
                      command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
                      command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
                      command.setWait(30);
                      final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
                      CheckRouterAnswer answer = null;
                      if (origAnswer instanceof CheckRouterAnswer) {
                          answer = (CheckRouterAnswer) origAnswer;
                      } else {
                          logger.warn("Unable to update router " + router.getHostName() + "'s status");
                      }
                      RedundantState state = RedundantState.UNKNOWN;
                      if (answer != null) {
                          if (answer.getResult()) {
                              state = answer.getState();
                          } else {
                              logger.info("Agent response doesn't seem to be correct ==> " + answer.getResult());
                          }
                      }
                      router.setRedundantState(state);
                      updated = true;
                  }
              }
              if (updated) {
                  _routerDao.update(router.getId(), router);
              }
              final RedundantState currState = router.getRedundantState();
              if (prevState != currState) {
                  final String title = "Redundant virtual router " + router.getInstanceName() + " just switch from " + prevState + " to " + currState;
                  final String context = String.format(
                          "Redundant virtual router [%s] with hostname: %s just switch from %s to %s",
                          router, router.getHostName(), prevState, currState);
                  logger.info(context);
                  if (currState == RedundantState.PRIMARY) {
                      _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
                  }
              }
          }
      }
  
      protected class RvRStatusUpdateTask extends ManagedContextRunnable {
  
          /*
           * In order to make fail-over works well at any time, we have to ensure:
           * 1. Backup router's priority = Primary's priority - DELTA + 1
           */
          private void checkSanity(final List<DomainRouterVO> routers) {
              final Set<Long> checkedNetwork = new HashSet<>();
              for (final DomainRouterVO router : routers) {
                  if (!router.getIsRedundantRouter()) {
                      continue;
                  }
  
                  final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
  
                  for (final Long routerGuestNtwkId : routerGuestNtwkIds) {
                      if (checkedNetwork.contains(routerGuestNtwkId)) {
                          continue;
                      }
                      checkedNetwork.add(routerGuestNtwkId);
  
                      final List<DomainRouterVO> checkingRouters;
                      final Long vpcId = router.getVpcId();
                      if (vpcId != null) {
                          checkingRouters = _routerDao.listByVpcId(vpcId);
                      } else {
                          checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER);
                      }
  
                      if (checkingRouters.size() != 2) {
                          continue;
                      }
  
                      DomainRouterVO primaryRouter = null;
                      DomainRouterVO backupRouter = null;
                      for (final DomainRouterVO r : checkingRouters) {
                          if (r.getRedundantState() == RedundantState.PRIMARY) {
                              if (primaryRouter == null) {
                                  primaryRouter = r;
                              } else {
                                  break;
                              }
                          } else if (r.getRedundantState() == RedundantState.BACKUP) {
                              if (backupRouter == null) {
                                  backupRouter = r;
                              } else {
                                  break;
                              }
                          }
                      }
                  }
              }
          }
  
          private void checkDuplicatePrimary(final List<DomainRouterVO> routers) {
              final Map<Long, DomainRouterVO> networkRouterMaps = new HashMap<>();
              for (final DomainRouterVO router : routers) {
                  final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
  
                  final Long vpcId = router.getVpcId();
                  if (vpcId != null || !routerGuestNtwkIds.isEmpty()) {
                      Long routerGuestNtwkId = vpcId != null ? vpcId : routerGuestNtwkIds.get(0);
                      if (router.getRedundantState() == RedundantState.PRIMARY) {
                          if (networkRouterMaps.containsKey(routerGuestNtwkId)) {
                              final DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId);
                              final String title = "More than one redundant virtual router is in PRIMARY state! Router " + router.getHostName() + " and router "
                                      + dupRouter.getHostName();
                              final String context = String.format(
                                      "Virtual router %s with hostname: %s and router %s " +
                                              "with hostname %s are both in PRIMARY state! " +
                                              "If the problem persist, restart both of routers. ",
                                      router, router.getHostName(), dupRouter, dupRouter.getHostName());
                              _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
                              logger.warn(context);
                          } else {
                              networkRouterMaps.put(routerGuestNtwkId, router);
                          }
                      }
                  }
              }
          }
  
          @Override
          protected void runInContext() {
              while (true) {
                  try {
                      final Long networkId = _vrUpdateQueue.take(); // This is a blocking call so this thread won't run all the time if no work item in queue.
  
                      final NetworkVO network = _networkDao.findById(networkId);
                      final Long vpcId = network.getVpcId();
  
                      final List<DomainRouterVO> routers;
                      if (vpcId != null) {
                          routers = _routerDao.listByVpcId(vpcId);
                      } else {
                          routers = _routerDao.listByNetworkAndRole(networkId, Role.VIRTUAL_ROUTER);
                      }
  
                      if (routers.size() != 2) {
                          continue;
                      }
                      /*
                       * We update the router pair which the lower id router owned
                       * by this mgmt server, in order to prevent duplicate update
                       * of router status from cluster mgmt servers
                       */
                      final DomainRouterVO router0 = routers.get(0);
                      final DomainRouterVO router1 = routers.get(1);
  
                      if (router0.getState() != VirtualMachine.State.Running || router1.getState() != VirtualMachine.State.Running) {
                          updateRoutersRedundantState(routers);
                          // Wilder Rodrigues (wrodrigues@schubergphilis.com) - One of the routers is not running,
                          // so we don't have to continue here since the host will be null any way. Also, there is no need
                          // To check either for sanity of duplicate primary. Thus, just update the state and get lost.
                          continue;
                      }
  
                      DomainRouterVO router = (router0.getId() < router1.getId()) ? router0 : router1;
                      // && router.getState() == VirtualMachine.State.Stopped
                      if (router.getHostId() == null && router.getState() == VirtualMachine.State.Running) {
                          logger.debug("Skip router pair (" + router0.getInstanceName() + "," + router1.getInstanceName() + ") due to can't find host");
                          continue;
                      }
                      final HostVO host = _hostDao.findById(router.getHostId());
                      if (host == null || host.getManagementServerId() == null || host.getManagementServerId() != ManagementServerNode.getManagementServerId()) {
                          logger.debug("Skip router pair (" + router0.getInstanceName() + "," + router1.getInstanceName() + ") due to not belong to this mgmt server");
                          continue;
                      }
                      updateRoutersRedundantState(routers);
                      checkDuplicatePrimary(routers);
                      checkSanity(routers);
                  } catch (final Exception ex) {
                      logger.error("Fail to complete the RvRStatusUpdateTask! ", ex);
                  }
              }
          }
      }
  
      protected class CheckRouterTask extends ManagedContextRunnable {
  
          public CheckRouterTask() {
          }
  
          @Override
          protected void runInContext() {
              try {
                  final List<DomainRouterVO> routers = _routerDao.listIsolatedByHostId(null);
                  logger.debug("Found " + routers.size() + " routers to update status. ");
  
                  updateSite2SiteVpnConnectionState(routers);
  
                  List<NetworkVO> networks = new ArrayList<>();
                  for (Vpc vpc : _vpcDao.listAll()) {
                      List<NetworkVO> vpcNetworks = _networkDao.listByVpc(vpc.getId());
                      if (!vpcNetworks.isEmpty()) {
                          networks.add(vpcNetworks.get(0));
                      }
                  }
                  logger.debug("Found " + networks.size() + " VPC's to update Redundant State. ");
                  pushToUpdateQueue(networks);
  
                  networks = _networkDao.listRedundantNetworks();
                  logger.debug("Found " + networks.size() + " networks to update RvR status. ");
                  pushToUpdateQueue(networks);
              } catch (final Exception ex) {
                  logger.error("Fail to complete the CheckRouterTask! ", ex);
              }
          }
  
          protected void pushToUpdateQueue(final List<NetworkVO> networks) throws InterruptedException {
              for (final NetworkVO network : networks) {
                  if (!_vrUpdateQueue.offer(network.getId(), 500, TimeUnit.MILLISECONDS)) {
                      logger.warn("Cannot insert into virtual router update queue! Adjustment of router.check.interval and router.check.poolsize maybe needed.");
                      break;
                  }
              }
          }
      }
  
      protected class FetchRouterHealthChecksResultTask extends ManagedContextRunnable {
          public FetchRouterHealthChecksResultTask() {
          }
  
          @Override
          protected void runInContext() {
              try {
                  final List<DomainRouterVO> routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId);
                  logger.info("Found " + routers.size() + " running routers. Fetching, analysing and updating DB for the health checks.");
                  if (!RouterHealthChecksEnabled.value()) {
                      logger.debug("Skipping fetching of router health check results as router.health.checks.enabled is disabled");
                      return;
                  }
  
                  for (final DomainRouterVO router : routers) {
                      GetRouterMonitorResultsAnswer answer = fetchAndUpdateRouterHealthChecks(router, false);
                      List<String> failingChecks = getFailingChecks(router, answer);
                      handleFailingChecks(router, failingChecks);
                  }
              } catch (final Exception ex) {
                  logger.error("Fail to complete the FetchRouterHealthChecksResultTask! ", ex);
              }
          }
      }
  
      private List<String> getFailingChecks(DomainRouterVO router, GetRouterMonitorResultsAnswer answer) {
  
          if (answer == null) {
              logger.warn("Unable to fetch monitor results for router {}", router);
              resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.UNKNOWN, RouterHealthStatus.UNKNOWN, "Communication failed");
              return List.of(CONNECTIVITY_TEST);
          } else if (!answer.getResult()) {
              logger.warn("Failed to fetch monitor results from router " + router + " with details: " + answer.getDetails());
              if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) {
                  resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.FAILED, "Failed to write: " + answer.getDetails());
                  return List.of(FILESYSTEM_WRITABLE_TEST);
              } else {
                  resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.FAILED, RouterHealthStatus.UNKNOWN, "Failed to fetch results with details: " + answer.getDetails());
                  return List.of(CONNECTIVITY_TEST);
              }
          } else {
              resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.SUCCESS, "Successfully fetched data");
              updateDbHealthChecksFromRouterResponse(router, answer.getMonitoringResults());
              return answer.getFailingChecks();
          }
      }
  
      private void handleFailingChecks(DomainRouterVO router, List<String> failingChecks) {
          if (CollectionUtils.isEmpty(failingChecks)) {
              return;
          }
  
          String alertMessage = String.format("Health checks failed: %d failing checks on router %s / %s", failingChecks.size(), router.getName(), router.getUuid());
          _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(),
                  alertMessage, alertMessage);
          logger.warn(alertMessage + ". Checking failed health checks to see if router needs recreate");
  
          String checkFailsToRecreateVr = RouterHealthChecksFailuresToRecreateVr.valueIn(router.getDataCenterId());
          StringBuilder failingChecksEvent = new StringBuilder();
          boolean recreateRouter = false;
          for (int i = 0; i < failingChecks.size(); i++) {
              String failedCheck = failingChecks.get(i);
              if (i == 0) {
                  failingChecksEvent.append("Router ")
                          .append(router.getName())
                          .append(" / ")
                          .append(router.getUuid())
                          .append(" has failing checks: ");
              }
  
              failingChecksEvent.append(failedCheck);
              if (i < failingChecks.size() - 1) {
                  failingChecksEvent.append(", ");
              }
  
              if (StringUtils.isNotBlank(checkFailsToRecreateVr) && checkFailsToRecreateVr.contains(failedCheck)) {
                  recreateRouter = true;
              }
          }
  
          ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,
                  Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS, failingChecksEvent.toString(), router.getId(), ApiCommandResourceType.DomainRouter.toString());
  
          if (recreateRouter) {
              logger.warn("Health Check Alert: Found failing checks in " +
                      RouterHealthChecksFailuresToRecreateVrCK + ", attempting recreating router.");
              recreateRouter(router);
          }
      }
  
      private DomainRouterJoinVO getAnyRouterJoinWithVpc(long routerId) {
          List<DomainRouterJoinVO> routerJoinVOs = domainRouterJoinDao.searchByIds(routerId);
          for (DomainRouterJoinVO router : routerJoinVOs) {
              if (router.getRemoved() == null && router.getVpcId() != 0) {
                  return router;
              }
          }
          return null;
      }
  
      private void restartVpcInDomainRouter(DomainRouterJoinVO router, User user) {
          try {
              logger.debug("Attempting restart VPC " + router.getVpcName() + " for router recreation " + router.getUuid());
              ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,
                      Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS,
                      "Recreating router " + router.getUuid() + " by restarting VPC " + router.getVpcUuid(), router.getId(), ApiCommandResourceType.DomainRouter.toString());
              vpcService.restartVpc(router.getVpcId(), true, false, false, user);
          } catch (Exception e) {
              logger.error("Failed to restart VPC for router recreation " +
                      router.getVpcName() + " ,router " + router.getUuid(), e);
          }
      }
  
      private DomainRouterJoinVO getAnyRouterJoinWithGuestTraffic(long routerId) {
          List<DomainRouterJoinVO> routerJoinVOs = domainRouterJoinDao.searchByIds(routerId);
          for (DomainRouterJoinVO router : routerJoinVOs) {
              if (router.getRemoved() == null && router.getTrafficType() == TrafficType.Guest) {
                  return router;
              }
          }
          return null;
      }
  
      private void restartGuestNetworkInDomainRouter(DomainRouterJoinVO router, User user) {
          try {
              logger.info("Attempting restart network " + router.getNetworkName() + " for router recreation " + router.getUuid());
              ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,
                      Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS,
                      "Recreating router " + router.getUuid() + " by restarting network " + router.getNetworkUuid(), router.getId(), ApiCommandResourceType.DomainRouter.toString());
              networkService.restartNetwork(router.getNetworkId(), true, false, false, user);
          } catch (Exception e) {
              logger.error("Failed to restart network " + router.getNetworkName() +
                      " for router recreation " + router.getNetworkName(), e);
          }
      }
  
      /**
       * Attempts recreation of router by restarting with cleanup a VPC if any or a guest network associated in case no VPC.
       * @param router - the router to be recreated.
       */
      private void recreateRouter(DomainRouterVO router) {
          long routerId = router.getId();
          User systemUser = _userDao.getUser(User.UID_SYSTEM);
  
          // Find any VPC containing router join VO, restart it and return
          DomainRouterJoinVO routerJoinToRestart = getAnyRouterJoinWithVpc(routerId);
          if (routerJoinToRestart != null) {
              restartVpcInDomainRouter(routerJoinToRestart, systemUser);
              return;
          }
  
          // If no VPC containing router join VO was found we look for a guest network traffic containing join VO and restart that.
          routerJoinToRestart = getAnyRouterJoinWithGuestTraffic(routerId);
          if (routerJoinToRestart != null) {
              restartGuestNetworkInDomainRouter(routerJoinToRestart, systemUser);
              return;
          }
  
          logger.warn("Unable to find a valid guest network or VPC to restart for recreating router {}", router);
      }
  
      private Map<String, Map<String, RouterHealthCheckResultVO>> getHealthChecksFromDb(long routerId) {
          List<RouterHealthCheckResultVO> healthChecksList = routerHealthCheckResultDao.getHealthCheckResults(routerId);
          Map<String, Map<String, RouterHealthCheckResultVO>> healthCheckResults = new HashMap<>();
          if (healthChecksList.isEmpty()) {
              return healthCheckResults;
          }
  
          for (RouterHealthCheckResultVO healthCheck : healthChecksList) {
              if (!healthCheckResults.containsKey(healthCheck.getCheckType())) {
                  healthCheckResults.put(healthCheck.getCheckType(), new HashMap<>());
              }
              healthCheckResults.get(healthCheck.getCheckType()).put(healthCheck.getCheckName(), healthCheck);
          }
  
          return healthCheckResults;
      }
  
      private void resetRouterHealthChecksAndConnectivity(final long routerId, VirtualNetworkApplianceService.RouterHealthStatus connected, VirtualNetworkApplianceService.RouterHealthStatus writable, String message) {
          routerHealthCheckResultDao.expungeHealthChecks(routerId);
          updateRouterHealthCheckResult(routerId, CONNECTIVITY_TEST, "basic", connected, connected.equals(RouterHealthStatus.SUCCESS) ? "Successfully connected to router" : message);
          updateRouterHealthCheckResult(routerId, FILESYSTEM_WRITABLE_TEST, "basic", writable, writable.equals(RouterHealthStatus.SUCCESS) ? "Successfully written to file system" : message);
      }
  
      private void updateRouterHealthCheckResult(final long routerId, String checkName, String checkType, VirtualNetworkApplianceService.RouterHealthStatus checkResult, String checkMessage) {
          boolean newHealthCheckEntry = false;
          RouterHealthCheckResultVO connectivityVO = routerHealthCheckResultDao.getRouterHealthCheckResult(routerId, checkName, checkType);
          if (connectivityVO == null) {
              connectivityVO = new RouterHealthCheckResultVO(routerId, checkName, checkType);
              newHealthCheckEntry = true;
          }
  
          connectivityVO.setCheckResult(checkResult);
          connectivityVO.setLastUpdateTime(new Date());
          if (StringUtils.isNotEmpty(checkMessage)) {
              connectivityVO.setCheckDetails(checkMessage.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
          }
  
          if (newHealthCheckEntry) {
              routerHealthCheckResultDao.persist(connectivityVO);
          } else {
              routerHealthCheckResultDao.update(connectivityVO.getId(), connectivityVO);
          }
      }
  
      private RouterHealthCheckResultVO parseHealthCheckVOFromJson(final long routerId,
                                                                   final String checkName, final String checkType, final Map<String, String> checkData,
                                                                   final Map<String, Map<String, RouterHealthCheckResultVO>> checksInDb) {
          RouterHealthStatus success = getRouterHealthStatus(checkData.get("success"));
          Date lastUpdate = new Date(Long.parseLong(checkData.get("lastUpdate")));
          double lastRunDuration = Double.parseDouble(checkData.get("lastRunDuration"));
          String message = checkData.get("message");
          final RouterHealthCheckResultVO hcVo;
          boolean newEntry = false;
          if (checksInDb.containsKey(checkType) && checksInDb.get(checkType).containsKey(checkName)) {
              hcVo = checksInDb.get(checkType).get(checkName);
          } else {
              hcVo = new RouterHealthCheckResultVO(routerId, checkName, checkType);
              newEntry = true;
          }
  
          hcVo.setCheckResult(success);
          hcVo.setLastUpdateTime(lastUpdate);
          if (StringUtils.isNotEmpty(message)) {
              hcVo.setCheckDetails(message.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
          }
  
          if (newEntry) {
              routerHealthCheckResultDao.persist(hcVo);
          } else {
              routerHealthCheckResultDao.update(hcVo.getId(), hcVo);
          }
          logger.info("Found health check " + hcVo + " which took running duration (ms) " + lastRunDuration);
          return hcVo;
      }
  
      private static RouterHealthStatus getRouterHealthStatus(String status) {
          RouterHealthStatus success;
          try {
              success = RouterHealthStatus.valueOf(status.trim());
          } catch (IllegalArgumentException | NullPointerException e) {
              success = RouterHealthStatus.UNKNOWN;
          }
          return success;
      }
  
      /**
       *
       * @param checksJson JSON expected is
       *                   {
       *                      checkType1: {
       *                          checkName1: {
       *                              success: true/false,
       *                              lastUpdate: date string,
       *                              lastRunDuration: ms spent on test,
       *                              message: detailed message from check execution
       *                          },
       *                          checkType2: .....
       *                      },
       *                      checkType2: ......
       *                   }
       * @return converts the above JSON into list of RouterHealthCheckResult.
       */
      private List<RouterHealthCheckResult> parseHealthCheckResults(
              final Map<String, Map<String, Map<String, String>>> checksJson, final DomainRouterVO router) {
          final Map<String, Map<String, RouterHealthCheckResultVO>> checksInDb = getHealthChecksFromDb(router.getId());
          List<RouterHealthCheckResult> healthChecks = new ArrayList<>();
          final String lastRunKey = "lastRun";
          for (String checkType : checksJson.keySet()) {
              if (checksJson.get(checkType).containsKey(lastRunKey)) { // Log last run of this check type run info
                  Map<String, String> lastRun = checksJson.get(checkType).get(lastRunKey);
                  logger.info("Found check types executed on VR " + checkType + ", start: " + lastRun.get("start") +
                          ", end: " + lastRun.get("end") + ", duration: " + lastRun.get("duration"));
              }
  
              for (String checkName : checksJson.get(checkType).keySet()) {
                  if (lastRunKey.equals(checkName)) {
                      continue;
                  }
  
                  try {
                      final RouterHealthCheckResultVO hcVo = parseHealthCheckVOFromJson(
                              router.getId(), checkName, checkType, checksJson.get(checkType).get(checkName), checksInDb);
                      healthChecks.add(hcVo);
                  } catch (Exception ex) {
                      logger.error("Skipping health check: Exception while parsing check result data for router {}, check type: {}, check name: {}:{}", router, checkType, checkName, ex.getLocalizedMessage(), ex);
                  }
              }
          }
          return healthChecks;
      }
  
      private void updateDbHealthChecksFromRouterResponse(final DomainRouterVO router, final String monitoringResult) {
          if (StringUtils.isBlank(monitoringResult)) {
              logger.warn("Attempted parsing empty monitoring results string for router {}", router);
              return;
          }
  
          try {
              logger.debug("Parsing and updating DB health check data for router: {} with data: {}", router, monitoringResult);
              final Type t = new TypeToken<Map<String, Map<String, Map<String, String>>>>() {}.getType();
              final Map<String, Map<String, Map<String, String>>> checks = GsonHelper.getGson().fromJson(monitoringResult, t);
              parseHealthCheckResults(checks, router);
          } catch (JsonSyntaxException ex) {
              logger.error("Unable to parse the result of health checks due to " + ex.getLocalizedMessage(), ex);
          }
      }
  
      private GetRouterMonitorResultsAnswer fetchAndUpdateRouterHealthChecks(DomainRouterVO router, boolean performFreshChecks) {
          if (!RouterHealthChecksEnabled.value()) {
              return null;
          }
  
          String controlIP = _routerControlHelper.getRouterControlIp(router.getId());
          if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) {
              final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(performFreshChecks, false);
              command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
              command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
              try {
                  final Answer answer = _agentMgr.easySend(router.getHostId(), command);
  
                  logger.debug("Got health check results from router {}: {}", router.getHostName(), answer != null ? answer.getDetails() : "null answer");
                  if (answer == null) {
                      logger.warn("Unable to fetch monitoring results data from router {}", router.getHostName());
                      return null;
                  }
                  if (answer instanceof GetRouterMonitorResultsAnswer) {
                      return (GetRouterMonitorResultsAnswer) answer;
                  } else {
                      logger.warn("Unable to fetch health checks results to router {} Received answer {}",  router.getHostName(), answer.getDetails());
                      return new GetRouterMonitorResultsAnswer(command, false, null, answer.getDetails());
                  }
              } catch (final Exception e) {
                  logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e);
                  return null;
              }
          }
  
          return null;
      }
  
      private GetRouterMonitorResultsAnswer performBasicTestsOnRouter(DomainRouterVO router) {
          if (!RouterHealthChecksEnabled.value()) {
              return null;
          }
  
          String controlIP = _routerControlHelper.getRouterControlIp(router.getId());
          if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) {
              final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(false, true);
              command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
              command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
              try {
                  final Answer answer = _agentMgr.easySend(router.getHostId(), command);
  
                  if (answer == null) {
                      logger.warn("Unable to fetch basic router test results data from router " + router.getHostName());
                      return null;
                  }
                  if (answer instanceof GetRouterMonitorResultsAnswer) {
                      return (GetRouterMonitorResultsAnswer) answer;
                  } else {
                      logger.warn("Unable to fetch basic router test results from router " + router.getHostName() + " Received answer " + answer.getDetails());
                      return new GetRouterMonitorResultsAnswer(command, false, null, answer.getDetails());
                  }
              } catch (final Exception e) {
                  logger.warn("Error while performing basic tests on router: " + router.getInstanceName(), e);
                  return null;
              }
          }
  
          return null;
      }
  
      @Override
      public Pair<Boolean, String> performRouterHealthChecks(long routerId) {
          DomainRouterVO router = _routerDao.findById(routerId);
  
          if (router == null) {
              throw new CloudRuntimeException("Unable to find router with id " + routerId);
          }
  
          if (!RouterHealthChecksEnabled.value()) {
              throw new CloudRuntimeException("Router health checks are not enabled for router: " + router);
          }
  
          logger.debug("Running health check results for router " + router.getUuid());
  
          GetRouterMonitorResultsAnswer answer;
          String resultDetails = "";
          boolean success = true;
  
          // Step 1: Perform basic tests to check the connectivity and file system on router
          answer = performBasicTestsOnRouter(router);
          if (answer == null) {
              logger.info("No results received for the basic tests on router: " + router);
              resultDetails = "Basic tests results unavailable";
              success = false;
          } else if (!answer.getResult()) {
              logger.warn("Basic tests failed on router: " + router);
              resultDetails = "Basic tests failed - " + answer.getMonitoringResults();
              success = false;
          } else {
              // Step 2: Update health check data on router and perform and retrieve health checks on router
              if (!updateRouterHealthChecksConfig(router)) {
                  logger.warn("Unable to update health check config for fresh run successfully for router: " + router + ", so trying to fetch last result.");
                  success = false;
                  answer = fetchAndUpdateRouterHealthChecks(router, false);
              } else {
                  logger.info("Successfully updated health check config for fresh run successfully for router: " + router);
                  answer = fetchAndUpdateRouterHealthChecks(router, true);
              }
  
              if (answer == null) {
                  resultDetails = "Failed to fetch and update health checks";
                  success = false;
              } else if (!answer.getResult()) {
                  resultDetails = "Get health checks failed - " + answer.getMonitoringResults();
                  success = false;
              }
          }
  
          // Step 3: Update health checks values in database. We do this irrespective of new health check config.
          List<String> failingChecks = getFailingChecks(router, answer);
          handleFailingChecks(router, failingChecks);
  
          return new Pair<>(success, resultDetails);
      }
  
      protected class UpdateRouterHealthChecksConfigTask extends ManagedContextRunnable {
          public UpdateRouterHealthChecksConfigTask() {
          }
  
          @Override
          protected void runInContext() {
              try {
                  final List<DomainRouterVO> routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId);
                  logger.debug("Found {} running routers. ", routers.size());
  
                  for (final DomainRouterVO router : routers) {
                      GetRouterMonitorResultsAnswer answer = performBasicTestsOnRouter(router);
                      if (answer != null && answer.getResult()) {
                          updateRouterHealthChecksConfig(router);
                      } else {
                          String resultDetails = (answer == null) ? "" : ", " + answer.getMonitoringResults();
                          logger.debug("Couldn't update health checks config on router: " + router + " as basic tests didn't succeed" + resultDetails);
                      }
                  }
              } catch (final Exception ex) {
                  logger.error("Fail to complete the UpdateRouterHealthChecksConfigTask! ", ex);
              }
          }
      }
  
      private SetMonitorServiceCommand createMonitorServiceCommand(DomainRouterVO router, List<MonitorServiceTO> services,
                                                                   boolean reconfigure, boolean deleteFromProcessedCache, Map<String, String> routerHealthCheckConfig) {
          final SetMonitorServiceCommand command = new SetMonitorServiceCommand(services);
          command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
          command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
          command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ENABLED, RouterHealthChecksEnabled.value().toString());
          command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_BASIC_INTERVAL, RouterHealthChecksBasicInterval.value().toString());
          command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL, RouterHealthChecksAdvancedInterval.value().toString());
          String excludedTests = RouterHealthChecksToExclude.valueIn(router.getDataCenterId());
          if (router.getIsRedundantRouter()) {
              // Disable gateway check if VPC has no tiers or no active VM's in it
              final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
              if (RedundantState.BACKUP.equals(router.getRedundantState()) ||
                      routerGuestNtwkIds == null || routerGuestNtwkIds.isEmpty()) {
                  excludedTests = excludedTests.isEmpty() ? BACKUP_ROUTER_EXCLUDED_TESTS : excludedTests + "," + BACKUP_ROUTER_EXCLUDED_TESTS;
              }
          }
  
          command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_EXCLUDED, excludedTests);
          command.setHealthChecksConfig(routerHealthCheckConfig);
          command.setReconfigureAfterUpdate(reconfigure);
          command.setDeleteFromProcessedCache(deleteFromProcessedCache); // As part of updating
          return command;
      }
  
      /**
       * Updates router health check config to the virtual router that it uses for health checks.
       * @param router - the router ID that data needs to be sent to.
       * @return success of whether data was sent or not
       */
      private boolean updateRouterHealthChecksConfig(DomainRouterVO router) {
          if (!RouterHealthChecksEnabled.value()) {
              return false;
          }
  
          String controlIP = _routerControlHelper.getRouterControlIp(router.getId());
          if (StringUtils.isBlank(controlIP) || controlIP.equals("0.0.0.0")) {
              logger.debug("Skipping update data on router " + router.getUuid() + " because controlIp is not correct.");
              return false;
          }
  
          logger.info("Updating data for router health checks for router " + router.getUuid());
          Answer origAnswer;
          try {
              SetMonitorServiceCommand command = createMonitorServiceCommand(router, null, true, true, getRouterHealthChecksConfig(router));
              origAnswer = _agentMgr.easySend(router.getHostId(), command);
          } catch (final Exception e) {
              logger.error("Error while sending update data for health check to router: " + router.getInstanceName(), e);
              return false;
          }
  
          if (origAnswer == null) {
              logger.error("Unable to update health checks data to router " + router.getHostName());
              return false;
          }
  
          GroupAnswer answer;
          if (origAnswer instanceof GroupAnswer) {
              answer = (GroupAnswer) origAnswer;
          } else {
              logger.error("Unable to update health checks data to router " + router.getHostName() + " Received answer " + origAnswer.getDetails());
              return false;
          }
  
          if (!answer.getResult()) {
              logger.error("Unable to update health checks data to router " + router.getHostName() + ", details : " + answer.getDetails());
          }
  
          return answer.getResult();
      }
  
      private String getSystemThresholdsHealthChecksData(final DomainRouterVO router) {
          return String.format("minDiskNeeded=%s,maxCpuUsage=%s,maxMemoryUsage=%s;",
                  RouterHealthChecksFreeDiskSpaceThreshold.valueIn(router.getDataCenterId()),
                  RouterHealthChecksMaxCpuUsageThreshold.valueIn(router.getDataCenterId()),
                  RouterHealthChecksMaxMemoryUsageThreshold.valueIn(router.getDataCenterId()));
      }
  
      private String getRouterVersionHealthChecksData(final DomainRouterVO router) {
          if (router.getTemplateVersion() != null && router.getScriptsVersion() != null) {
              return String.format("templateVersion=%s,scriptsVersion=%s", router.getTemplateVersion(),
                      router.getScriptsVersion());
          }
          return null;
      }
  
      private void updateWithPortForwardingRules(final DomainRouterJoinVO routerJoinVO, final UserVmJoinVO vm, final StringBuilder portData) {
          SearchBuilder<PortForwardingRuleVO> sbpf = portForwardingDao.createSearchBuilder();
          sbpf.and("networkId", sbpf.entity().getNetworkId(), SearchCriteria.Op.EQ);
          sbpf.and("instanceId", sbpf.entity().getVirtualMachineId(), SearchCriteria.Op.EQ);
          SearchCriteria<PortForwardingRuleVO> scpf = sbpf.create();
          scpf.setParameters("networkId", routerJoinVO.getNetworkId());
          scpf.setParameters("instanceId", vm.getId());
          List<PortForwardingRuleVO> portForwardingRules = portForwardingDao.search(scpf, null);
          for (PortForwardingRuleVO portForwardingRule : portForwardingRules) {
              portData.append("sourceIp=").append(_ipAddressDao.findById(portForwardingRule.getSourceIpAddressId()).getAddress().toString())
                      .append(",sourcePortStart=").append(portForwardingRule.getSourcePortStart())
                      .append(",sourcePortEnd=").append(portForwardingRule.getSourcePortEnd())
                      .append(",destIp=").append(portForwardingRule.getDestinationIpAddress())
                      .append(",destPortStart=").append(portForwardingRule.getDestinationPortStart())
                      .append(",destPortEnd=").append(portForwardingRule.getDestinationPortEnd()).append(";");
          }
      }
  
      private String getStickinessPolicies(long loadBalancingRuleId) {
          List<LBStickinessPolicyVO> stickinessPolicyVOs = lbStickinessPolicyDao.listByLoadBalancerId(loadBalancingRuleId, false);
          if (stickinessPolicyVOs != null && !stickinessPolicyVOs.isEmpty()) {
              StringBuilder stickiness = new StringBuilder();
              for (LBStickinessPolicyVO stickinessVO : stickinessPolicyVOs) {
                  stickiness.append(stickinessVO.getMethodName()).append(" ");
              }
              return stickiness.toString().trim();
          }
          return "None";
      }
  
      private void updateWithLbRules(final DomainRouterJoinVO routerJoinVO, final StringBuilder loadBalancingData) {
          List<? extends FirewallRuleVO> loadBalancerVOs = this.getLBRules(routerJoinVO);
          for (FirewallRuleVO firewallRuleVO : loadBalancerVOs) {
              List<LoadBalancerVMMapVO> vmMapVOs = _loadBalancerVMMapDao.listByLoadBalancerId(firewallRuleVO.getId(), false);
              if (!vmMapVOs.isEmpty()) {
  
                  final NetworkOffering offering = _networkOfferingDao.findById(_networkDao.findById(routerJoinVO.getNetworkId()).getNetworkOfferingId());
                  if (offering.getConcurrentConnections() == null) {
                      loadBalancingData.append("maxconn=").append(NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value());
                  } else {
                      loadBalancingData.append("maxconn=").append(offering.getConcurrentConnections());
                  }
  
                  loadBalancingData.append(",sourcePortStart=").append(firewallRuleVO.getSourcePortStart())
                          .append(",sourcePortEnd=").append(firewallRuleVO.getSourcePortEnd());
                  if (firewallRuleVO instanceof LoadBalancerVO) {
                      LoadBalancerVO loadBalancerVO = (LoadBalancerVO) firewallRuleVO;
                      String sourceIp = _ipAddressDao.findById(loadBalancerVO.getSourceIpAddressId()).getAddress().toString();
                      loadBalancingData.append(",sourceIp=").append(sourceIp)
                              .append(",destPortStart=").append(loadBalancerVO.getDefaultPortStart())
                              .append(",destPortEnd=").append(loadBalancerVO.getDefaultPortEnd())
                              .append(",algorithm=").append(loadBalancerVO.getAlgorithm())
                              .append(",protocol=").append(loadBalancerVO.getLbProtocol());
                      updateWithLbRuleSslCertificates(loadBalancingData, loadBalancerVO, sourceIp);
                  } else if (firewallRuleVO instanceof ApplicationLoadBalancerRuleVO) {
                      ApplicationLoadBalancerRuleVO appLoadBalancerVO = (ApplicationLoadBalancerRuleVO) firewallRuleVO;
                      loadBalancingData.append(",sourceIp=").append(appLoadBalancerVO.getSourceIp())
                              .append(",destPortStart=").append(appLoadBalancerVO.getDefaultPortStart())
                              .append(",destPortEnd=").append(appLoadBalancerVO.getDefaultPortEnd())
                              .append(",algorithm=").append(appLoadBalancerVO.getAlgorithm())
                              .append(",protocol=").append(appLoadBalancerVO.getLbProtocol());
                  }
                  loadBalancingData.append(",stickiness=").append(getStickinessPolicies(firewallRuleVO.getId()));
                  loadBalancingData.append(",keepAliveEnabled=").append(offering.isKeepAliveEnabled()).append(",vmIps=");
                  for (LoadBalancerVMMapVO vmMapVO : vmMapVOs) {
                      loadBalancingData.append(vmMapVO.getInstanceIp()).append(" ");
                  }
                  loadBalancingData.setCharAt(loadBalancingData.length() - 1, ';');
              }
          }
      }
  
      protected void updateWithLbRuleSslCertificates(final StringBuilder loadBalancingData, LoadBalancerVO loadBalancerVO, String sourceIp) {
          if (NetUtils.SSL_PROTO.equals(loadBalancerVO.getLbProtocol())) {
              final LbSslCert sslCert = _lbMgr.getLbSslCert(loadBalancerVO.getId());
              if (sslCert != null && ! sslCert.isRevoked()) {
                  loadBalancingData.append(",sslcert=").append(sourceIp.replace(".", "_")).append('-')
                          .append(loadBalancerVO.getSourcePortStart()).append(".pem");
              }
          }
      }
  
      protected Map<String, String> getRouterHealthChecksConfig(final DomainRouterVO router) {
          Map<String, String> data = new HashMap<>();
          List<DomainRouterJoinVO> routerJoinVOs = domainRouterJoinDao.searchByIds(router.getId());
          StringBuilder vmsData = new StringBuilder();
          StringBuilder portData = new StringBuilder();
          StringBuilder loadBalancingData = new StringBuilder();
          StringBuilder gateways = new StringBuilder();
          gateways.append("gatewaysIps=");
          for (DomainRouterJoinVO routerJoinVO : routerJoinVOs) {
              if (StringUtils.isNotBlank(routerJoinVO.getGateway())) {
                  gateways.append(routerJoinVO.getGateway()).append(" ");
              }
              SearchBuilder<UserVmJoinVO> sbvm = userVmJoinDao.createSearchBuilder();
              sbvm.and("networkId", sbvm.entity().getNetworkId(), SearchCriteria.Op.EQ);
              sbvm.and("state", sbvm.entity().getState(), SearchCriteria.Op.EQ);
              SearchCriteria<UserVmJoinVO> scvm = sbvm.create();
              scvm.setParameters("networkId", routerJoinVO.getNetworkId());
              scvm.setParameters("state", VirtualMachine.State.Running);
              List<UserVmJoinVO> vms = userVmJoinDao.search(scvm, null);
              boolean isDhcpSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp);
              boolean isDnsSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dns);
              for (UserVmJoinVO vm : vms) {
                  vmsData.append("vmName=").append(vm.getName())
                          .append(",macAddress=").append(vm.getMacAddress())
                          .append(",ip=").append(vm.getIpAddress())
                          .append(",dhcp=").append(isDhcpSupported)
                          .append(",dns=").append(isDnsSupported).append(";");
                  updateWithPortForwardingRules(routerJoinVO, vm, portData);
              }
              updateWithLbRules(routerJoinVO, loadBalancingData);
          }
  
          String routerVersion = getRouterVersionHealthChecksData(router);
          data.put("virtualMachines", vmsData.toString());
          data.put("gateways", gateways.toString());
          data.put("portForwarding", portData.toString());
          data.put("haproxyData", loadBalancingData.toString());
          data.put("systemThresholds", getSystemThresholdsHealthChecksData(router));
          if (routerVersion != null) {
              data.put("routerVersion", routerVersion);
          }
          return data;
      }
  
      private List<? extends FirewallRuleVO> getLBRules(final DomainRouterJoinVO router) {
          if (router.getRole() == Role.VIRTUAL_ROUTER) {
              SearchBuilder<LoadBalancerVO> sblb = _loadBalancerDao.createSearchBuilder();
              sblb.and("networkId", sblb.entity().getNetworkId(), SearchCriteria.Op.EQ);
              sblb.and("sourceIpAddressId", sblb.entity().getSourceIpAddressId(), SearchCriteria.Op.NNULL);
              SearchCriteria<LoadBalancerVO> sclb = sblb.create();
              sclb.setParameters("networkId", router.getNetworkId());
              return _loadBalancerDao.search(sclb, null);
          } else if (router.getRole() == Role.INTERNAL_LB_VM) {
              SearchBuilder<ApplicationLoadBalancerRuleVO> sbalb = applicationLoadBalancerRuleDao.createSearchBuilder();
              sbalb.and("networkId", sbalb.entity().getNetworkId(), SearchCriteria.Op.EQ);
              sbalb.and("sourceIpAddress", sbalb.entity().getSourceIp(), SearchCriteria.Op.NNULL);
              SearchCriteria<ApplicationLoadBalancerRuleVO> sclb = sbalb.create();
              sclb.setParameters("networkId", router.getNetworkId());
              return applicationLoadBalancerRuleDao.search(sclb, null);
          }
          return Collections.emptyList();
      }
  
      protected class CheckRouterAlertsTask extends ManagedContextRunnable {
          public CheckRouterAlertsTask() {
          }
  
          @Override
          protected void runInContext() {
              try {
                  getRouterAlerts();
              } catch (final Exception ex) {
                  logger.error("Fail to complete the CheckRouterAlertsTask! ", ex);
              }
          }
      }
  
      protected void getRouterAlerts() {
          try {
              final List<DomainRouterVO> routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId);
  
              logger.debug("Found " + routers.size() + " running routers. ");
              for (final DomainRouterVO router : routers) {
                  final Boolean serviceMonitoringFlag = SetServiceMonitor.valueIn(router.getDataCenterId());
                  // Skip the routers in VPC network or skip the routers where
                  // Monitor service is not enabled in the corresponding Zone
                  if (serviceMonitoringFlag == null || !serviceMonitoringFlag) {
                      continue;
                  }
                  String controlIP = _routerControlHelper.getRouterControlIp(router.getId());
  
                  if (controlIP != null && !controlIP.equals("0.0.0.0")) {
                      OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId());
  
                      GetRouterAlertsCommand command = getGetRouterAlertsCommand(opRouterMonitorServiceVO, controlIP);
  
                      try {
                          final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
                          GetRouterAlertsAnswer answer;
  
                          if (origAnswer == null) {
                              logger.warn("Unable to get alerts from router " + router.getHostName());
                              continue;
                          }
                          if (origAnswer instanceof GetRouterAlertsAnswer) {
                              answer = (GetRouterAlertsAnswer) origAnswer;
                          } else {
                              logger.warn("Unable to get alerts from router " + router.getHostName());
                              continue;
                          }
                          if (!answer.getResult()) {
                              logger.warn("Unable to get alerts from router " + router.getHostName() + " " + answer.getDetails());
                              continue;
                          }
  
                          final String[] alerts = answer.getAlerts();
                          if (alerts != null) {
                              final String lastAlertTimeStamp = answer.getTimeStamp();
                              final SimpleDateFormat sdfrmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                              sdfrmt.setLenient(false);
                              try {
                                  sdfrmt.parse(lastAlertTimeStamp);
                              } catch (final ParseException e) {
                                  logger.warn("Invalid last alert timestamp received while collecting alerts from router: " + router.getInstanceName());
                                  continue;
                              }
                              for (final String alert : alerts) {
                                  _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), "Monitoring Service on VR "
                                          + router.getInstanceName(), alert);
                              }
                              if (opRouterMonitorServiceVO == null) {
                                  opRouterMonitorServiceVO = new OpRouterMonitorServiceVO(router.getId(), router.getHostName(), lastAlertTimeStamp);
                                  _opRouterMonitorServiceDao.persist(opRouterMonitorServiceVO);
                              } else {
                                  opRouterMonitorServiceVO.setLastAlertTimestamp(lastAlertTimeStamp);
                                  _opRouterMonitorServiceDao.update(opRouterMonitorServiceVO.getId(), opRouterMonitorServiceVO);
                              }
                          }
                      } catch (final Exception e) {
                          logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e);
                      }
                  }
              }
          } catch (final Exception e) {
              logger.warn("Error while collecting alerts from router", e);
          }
      }
  
      private static GetRouterAlertsCommand getGetRouterAlertsCommand(OpRouterMonitorServiceVO opRouterMonitorServiceVO, String controlIP) {
          GetRouterAlertsCommand command;
          if (opRouterMonitorServiceVO == null) {
              command = new GetRouterAlertsCommand("1970-01-01 00:00:00"); // To avoid sending null value
          } else {
              command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp());
          }
  
          command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
          return command;
      }
  
      @Override
      public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) {
  
          boolean dnsProvided = true;
          boolean dhcpProvided = true;
          boolean publicNetwork = false;
          final DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId());
          _dcDao.loadDetails(dc);
  
          // 1) Set router details
          final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId());
          final Map<String, String> details = _vmDetailsDao.listDetailsKeyPairs(router.getId());
          router.setDetails(details);
  
          // 2) Prepare bootloader elements related with Control network
  
          final StringBuilder buf = profile.getBootArgsBuilder();
          buf.append(" template=domP");
          buf.append(" name=").append(profile.getHostName());
  
          if (Boolean.parseBoolean(_configDao.getValue("system.vm.random.password"))) {
              buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password"));
          }
          String msPublicKey = _configDao.getValue("ssh.publickey");
          buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey));
  
          NicProfile controlNic = null;
          String defaultDns1 = null;
          String defaultDns2 = null;
          String defaultIp6Dns1 = null;
          String defaultIp6Dns2 = null;
          for (final NicProfile nic : profile.getNics()) {
              final int deviceId = nic.getDeviceId();
              boolean ipv4 = false, ipv6 = false;
              if (nic.getIPv4Address() != null) {
                  ipv4 = true;
                  buf.append(" eth").append(deviceId).append("ip=").append(nic.getIPv4Address());
                  buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask());
              }
              if (nic.getIPv6Address() != null) {
                  ipv6 = true;
                  buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIPv6Address());
                  buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIPv6Cidr()));
              }
  
              if (nic.isDefaultNic()) {
                  if (ipv4) {
                      buf.append(" gateway=").append(nic.getIPv4Gateway());
                  }
                  if (ipv6) {
                      defaultIp6Dns1 = nic.getIPv6Dns1();
                      defaultIp6Dns2 = nic.getIPv6Dns2();
                      buf.append(" ip6gateway=").append(nic.getIPv6Gateway());
                  }
                  defaultDns1 = nic.getIPv4Dns1();
                  defaultDns2 = nic.getIPv4Dns2();
              }
  
              if (nic.getTrafficType() == TrafficType.Management) {
                  buf.append(" localgw=").append(dest.getPod().getGateway());
              } else if (nic.getTrafficType() == TrafficType.Control) {
                  controlNic = nic;
                  buf.append(createRedundantRouterArgs(controlNic, router));
  
                  // DOMR control command is sent over management server in VMware
                  if (dest.getHost().getHypervisorType() == HypervisorType.VMware || dest.getHost().getHypervisorType() == HypervisorType.Hyperv) {
                      logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " + dest.getPod().getCidrAddress() + "/"
                              + dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: "
                              + ApiServiceConfiguration.ManagementServerAddresses.value());
  
                      if (logger.isInfoEnabled()) {
                          logger.info("Add management server explicit route to DomR.");
                      }
  
                      // always add management explicit route, for basic
                      // networking setup, DomR may have two interfaces while both
                      // are on the same subnet
                      String _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key());
                      if (NetUtils.isValidIp4Cidr(_mgmtCidr)) {
                          buf.append(" mgmtcidr=").append(_mgmtCidr);
                          buf.append(" localgw=").append(dest.getPod().getGateway());
                      }
  
                      if (dc.getNetworkType() == NetworkType.Basic) {
                          // ask domR to set up SSH on guest network
                          if (profile.getHypervisorType() == HypervisorType.VMware) {
                              buf.append(" sshonguest=false");
                          } else {
                              buf.append(" sshonguest=true");
                          }
                      }
  
                  }
              } else if (nic.getTrafficType() == TrafficType.Guest) {
                  logger.info("Guest IP : " + nic.getIPv4Address());
                  dnsProvided = _networkModel.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter);
                  dhcpProvided = _networkModel.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter);
                  buf.append(" privateMtu=").append(nic.getMtu());
                  // build bootloader parameter for the guest
                  buf.append(createGuestBootLoadArgs(nic, router));
              } else if (nic.getTrafficType() == TrafficType.Public) {
                  logger.info("Public IP : " + nic.getIPv4Address());
                  publicNetwork = true;
                  buf.append(" publicMtu=").append(nic.getMtu());
              }
          }
  
          if (controlNic == null) {
              throw new CloudRuntimeException("Didn't start a control port");
          }
  
          final String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key());
          _disableRpFilter = rpValue != null && rpValue.equalsIgnoreCase("true");
  
          String rpFilter = " ";
          String type;
          if (router.getVpcId() != null) {
              type = "vpcrouter";
              if (_disableRpFilter) {
                  rpFilter = " disable_rp_filter=true";
              }
              Vpc vpc = vpcManager.getActiveVpc(router.getVpcId());
              if (vpcManager.isSrcNatIpRequiredForVpcVr(vpc.getVpcOfferingId())) {
                  buf.append(" has_public_network=true");
              } else {
                  buf.append(" has_public_network=false");
              }
          } else if (!publicNetwork) {
              type = "dhcpsrvr";
          } else {
              type = "router";
              if (_disableRpFilter) {
                  rpFilter = " disable_rp_filter=true";
              }
          }
  
          if (_disableRpFilter) {
              rpFilter = " disable_rp_filter=true";
          }
  
          buf.append(" type=").append(type).append(rpFilter);
  
          final String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName());
          if (domain_suffix != null) {
              buf.append(" dnssearchorder=").append(domain_suffix);
          }
  
          if (profile.getHypervisorType() == HypervisorType.Hyperv) {
              buf.append(" extra_pubnics=").append(_routerExtraPublicNics);
          }
  
          /*
           * If virtual router didn't provide DNS service but provide DHCP
           * service, we need to override the DHCP response to return DNS server
           * rather than virtual router itself.
           */
          boolean useRouterIpResolver = getUseRouterIpAsResolver(router);
          if (dnsProvided || dhcpProvided) {
              if (defaultDns1 != null) {
                  buf.append(" dns1=").append(defaultDns1);
              }
              if (defaultDns2 != null) {
                  buf.append(" dns2=").append(defaultDns2);
              }
              if (defaultIp6Dns1 != null) {
                  buf.append(" ip6dns1=").append(defaultIp6Dns1);
              }
              if (defaultIp6Dns2 != null) {
                  buf.append(" ip6dns2=").append(defaultIp6Dns2);
              }
  
              boolean useExtDns = !dnsProvided;
              /* For backward compatibility */
              useExtDns = useExtDns || UseExternalDnsServers.valueIn(dc.getId());
  
              if (useExtDns) {
                  buf.append(" useextdns=true");
              }
              if (useRouterIpResolver) {
                  buf.append(" userouteripresolver=true");
              }
          }
  
          if (Boolean.TRUE.equals(ExposeDnsAndBootpServer.valueIn(dc.getId()))) {
              buf.append(" exposedns=true");
          }
  
          if (Boolean.parseBoolean(_configDao.getValue(Config.BaremetalProvisionDoneNotificationEnabled.key()))) {
              final QueryBuilder<UserVO> acntq = QueryBuilder.create(UserVO.class);
              acntq.and(acntq.entity().getUsername(), SearchCriteria.Op.EQ, "baremetal-system-account");
              final UserVO user = acntq.find();
              if (user == null) {
                  logger.warn("global setting[baremetal.provision.done.notification] is enabled but user baremetal-system-account is not found. Baremetal provision done notification will not be enabled");
              } else {
                  buf.append(String.format(" baremetalnotificationsecuritykey=%s", user.getSecretKey()));
                  buf.append(String.format(" baremetalnotificationapikey=%s", user.getApiKey()));
                  buf.append(" host=").append(ApiServiceConfiguration.ManagementServerAddresses.value());
                  buf.append(" port=").append(_configDao.getValue(Config.BaremetalProvisionDoneNotificationPort.key()));
              }
          }
  
          String routerLogrotateFrequency = RouterLogrotateFrequency.valueIn(router.getDataCenterId());
          if (!checkLogrotateTimerPattern(routerLogrotateFrequency)) {
              logger.debug(String.format("Setting [%s] with value [%s] do not match with the used regex [%s], or any acceptable value ('hourly', 'daily', 'monthly'); " +
                              "therefore, we will use the default value [%s] to configure the logrotate service on the virtual router.",RouterLogrotateFrequency.key(),
                      routerLogrotateFrequency, loggerROTATE_REGEX, RouterLogrotateFrequency.defaultValue()));
              routerLogrotateFrequency = RouterLogrotateFrequency.defaultValue();
          }
          logger.debug(String.format("The setting [%s] with value [%s] for the zone with UUID [%s], will be used to configure the logrotate service frequency" +
                  " on the virtual router.", RouterLogrotateFrequency.key(), routerLogrotateFrequency, dc.getUuid()));
          buf.append(String.format(" logrotatefrequency=%s", routerLogrotateFrequency));
  
          if (SystemVmEnableUserData.valueIn(router.getDataCenterId())) {
              String userDataUuid = VirtualRouterUserData.valueIn(dc.getId());
              try {
                  String userData = userDataManager.validateAndGetUserDataForSystemVM(userDataUuid);
                  if (StringUtils.isNotBlank(userData)) {
                      buf.append(" userdata=").append(userData);
                  }
              } catch (Exception e) {
                  logger.warn("Failed to load user data for the virtual router, ignored", e);
              }
          }
  
          if (logger.isDebugEnabled()) {
              logger.debug("Boot Args for " + profile + ": " + buf);
          }
  
          return true;
      }
  
      private boolean getUseRouterIpAsResolver(DomainRouterVO router) {
          if (router == null || router.getVpcId() == null) {
              return false;
          }
          Vpc vpc = _vpcDao.findById(router.getVpcId());
          if (vpc == null) {
              logger.warn(String.format("Cannot find VPC with ID %s from router %s", router.getVpcId(), router.getName()));
              return false;
          }
          return vpc.useRouterIpAsResolver();
      }
  
      /**
       * @param routerLogrotateFrequency The string to be checked if matches with any acceptable values.
       * Checks if the value in the global configuration is an acceptable value to be informed to the Virtual Router.
       * @return true if the passed value match with any acceptable value based on the regex ((?i)(hourly)|(daily)|(monthly))|(\*|\d{2})\:(\*|\d{2})\:(\*|\d{2})
       */
      protected boolean checkLogrotateTimerPattern(String routerLogrotateFrequency) {
          return Pattern.matches(loggerROTATE_REGEX, routerLogrotateFrequency);
      }
  
      protected StringBuilder createGuestBootLoadArgs(final NicProfile guestNic, final DomainRouterVO router) {
          final long guestNetworkId = guestNic.getNetworkId();
          final NetworkVO guestNetwork = _networkDao.findById(guestNetworkId);
          String dhcpRange = null;
          final DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId());
  
          final StringBuilder buf = new StringBuilder();
  
          final NetworkOffering offering = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId());
          boolean ipv4Routed = NetworkOffering.NetworkMode.ROUTED.equals(offering.getNetworkMode());
          if (ipv4Routed) {
              buf.append(" is_routed=true");
          }
  
          boolean isIpv6Supported = _networkOfferingDao.isIpv6Supported(guestNetwork.getNetworkOfferingId());
          if (isIpv6Supported) {
              buf.append(" ip6firewall=true");
          }
  
          final boolean isRedundant = router.getIsRedundantRouter();
          if (isRedundant) {
              buf.append(createRedundantRouterArgs(guestNic, router));
              final Network net = _networkModel.getNetwork(guestNic.getNetworkId());
              buf.append(" guestgw=").append(net.getGateway());
              if (ObjectUtils.allNotNull(net.getIp6Gateway(), guestNic.getIPv6Cidr())) {
                  buf.append(" guestgw6=").append(net.getIp6Gateway());
                  buf.append(" guestcidr6size=").append(NetUtils.getIp6CidrSize(guestNic.getIPv6Cidr()));
              }
              final String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIPv4Address()) | ~NetUtils.ip2Long(guestNic.getIPv4Netmask()));
              buf.append(" guestbrd=").append(brd);
              buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getIPv4Netmask()));
  
              final int advertInt = NumbersUtil.parseInt(_configDao.getValue(Config.RedundantRouterVrrpInterval.key()), 1);
              buf.append(" advert_int=").append(advertInt);
          }
  
          // setup network domain
          final String domain = guestNetwork.getNetworkDomain();
          if (domain != null) {
              buf.append(" domain=").append(domain);
          }
  
          long cidrSize = 0;
  
          // setup dhcp range
          if (dc.getNetworkType() == NetworkType.Basic) {
              if (guestNic.isDefaultNic()) {
                  cidrSize = NetUtils.getCidrSize(guestNic.getIPv4Netmask());
                  final String cidr = NetUtils.getCidrSubNet(guestNic.getIPv4Gateway(), cidrSize);
                  dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize);
              }
          } else if (dc.getNetworkType() == NetworkType.Advanced) {
              final String cidr = _networkModel.getValidNetworkCidr(guestNetwork);
              if (cidr != null) {
                  cidrSize = NetUtils.getCidrSize(NetUtils.getCidrNetmask(cidr));
                  dhcpRange = NetUtils.getDhcpRange(cidr);
              }
          }
  
          if (dhcpRange != null) {
              // To limit DNS to the cidr range
              buf.append(" cidrsize=").append(cidrSize);
              buf.append(" dhcprange=").append(dhcpRange);
          }
  
          return buf;
      }
  
      protected StringBuilder createRedundantRouterArgs(final NicProfile nic, final DomainRouterVO router) {
          final StringBuilder buf = new StringBuilder();
  
          final boolean isRedundant = router.getIsRedundantRouter();
          if (isRedundant) {
              buf.append(" redundant_router=1");
  
              final int advertInt = NumbersUtil.parseInt(_configDao.getValue(Config.RedundantRouterVrrpInterval.key()), 1);
              buf.append(" advert_int=").append(advertInt);
  
              final Long vpcId = router.getVpcId();
              final List<DomainRouterVO> routers;
              if (vpcId != null) {
                  routers = _routerDao.listByVpcId(vpcId);
                  // For a redundant VPC router, both shall have the same router id. It will be used by the VRRP virtural_router_id attribute.
                  // So we use the VPC id to avoid group problems.
                  buf.append(" router_id=").append(vpcId);
  
                  // Will build the routers password based on the VPC ID and UUID.
                  final Vpc vpc = _vpcDao.findById(vpcId);
  
                  try {
                      final MessageDigest digest = MessageDigest.getInstance("SHA-512");
                      final byte [] rawDigest = vpc.getUuid().getBytes(Charset.defaultCharset());
                      digest.update(rawDigest);
  
                      final BigInteger password = new BigInteger(1, digest.digest());
                      buf.append(" router_password=").append(password);
  
                  } catch (final NoSuchAlgorithmException e) {
                      logger.error("Failed to pssword! Will use the plan B instead.");
                      buf.append(" router_password=").append(vpc.getUuid());
                  }
  
              } else {
                  routers = _routerDao.listByNetworkAndRole(nic.getNetworkId(), Role.VIRTUAL_ROUTER);
              }
  
              String redundantState = RedundantState.BACKUP.toString();
              router.setRedundantState(RedundantState.BACKUP);
              if (routers.isEmpty()) {
                  redundantState = RedundantState.PRIMARY.toString();
                  router.setRedundantState(RedundantState.PRIMARY);
              } else {
                  final DomainRouterVO router0 = routers.get(0);
                  if (router.getId() == router0.getId()) {
                      redundantState = RedundantState.PRIMARY.toString();
                      router.setRedundantState(RedundantState.PRIMARY);
                  }
              }
  
              buf.append(" redundant_state=").append(redundantState);
          }
  
          return buf;
      }
  
      @Override
      public boolean finalizeDeployment(final Commands cmds, final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context)
              throws ResourceUnavailableException {
          final DomainRouterVO router = _routerDao.findById(profile.getId());
  
          final List<NicProfile> nics = profile.getNics();
          for (final NicProfile nic : nics) {
              if (nic.getTrafficType() == TrafficType.Public) {
                  router.setPublicIpAddress(nic.getIPv4Address());
                  router.setPublicNetmask(nic.getIPv4Netmask());
                  router.setPublicMacAddress(nic.getMacAddress());
              } else if (nic.getTrafficType() == TrafficType.Control) {
                  router.setPrivateIpAddress(nic.getIPv4Address());
                  router.setPrivateMacAddress(nic.getMacAddress());
              }
          }
          _routerDao.update(router.getId(), router);
  
          finalizeCommandsOnStart(cmds, profile);
          return true;
      }
  
      private Provider getVrProvider(DomainRouterVO router) {
          final VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId());
          if (vrProvider == null) {
              throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName());
          }
          final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString());
          if (provider == null) {
              throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString());
          }
          return provider;
      }
  
      @Override
      public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachineProfile profile) {
          final DomainRouterVO router = _routerDao.findById(profile.getId());
          final NicProfile controlNic = getControlNic(profile);
  
          if (controlNic == null) {
              logger.error("Control network doesn't exist for the router " + router);
              return false;
          }
  
          finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic);
  
          // restart network if restartNetwork = false is not specified in profile
          // parameters
          boolean reprogramGuestNtwks = !Boolean.FALSE.equals(profile.getParameter(Param.ReProgramGuestNetworks));
  
          final Provider provider = getVrProvider(router);
  
          final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
          Map <String, String> routerHealthChecksConfig = getRouterHealthChecksConfig(router);
          for (final Long guestNetworkId : routerGuestNtwkIds) {
              final AggregationControlCommand startCmd = new AggregationControlCommand(Action.Start, router.getInstanceName(), controlNic.getIPv4Address(), _routerControlHelper.getRouterIpInNetwork(
                      guestNetworkId, router.getId()));
              cmds.addCommand(startCmd);
  
              if (reprogramGuestNtwks) {
                  finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null);
                  finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId);
                  finalizeMonitorService(cmds, profile, router, provider, guestNetworkId, true, routerHealthChecksConfig);
              }
  
              finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId);
  
              final AggregationControlCommand finishCmd = new AggregationControlCommand(Action.Finish, router.getInstanceName(), controlNic.getIPv4Address(), _routerControlHelper.getRouterIpInNetwork(
                      guestNetworkId, router.getId()));
              cmds.addCommand(finishCmd);
          }
  
          return true;
      }
  
      protected void finalizeMonitorService(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final Provider provider,
                                            final long networkId, boolean onStart, Map<String, String> routerHealthCheckConfig) {
          final NetworkOffering offering = _networkOfferingDao.findById(_networkDao.findById(networkId).getNetworkOfferingId());
          if (offering.isRedundantRouter()) {
              // service monitoring is currently not added in RVR
              return;
          }
  
          final String serviceMonitoringSet = _configDao.getValue(Config.EnableServiceMonitoring.key());
          final boolean isMonitoringServicesEnabled = serviceMonitoringSet != null && serviceMonitoringSet.equalsIgnoreCase("true");
          final NetworkVO network = _networkDao.findById(networkId);
  
          logger.debug("Creating  monitoring services on " + router + " start...");
  
          // get the list of sevices for this network to monitor
          final List<MonitoringServiceVO> services = new ArrayList<>();
          if (_networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dhcp, provider)
                  || _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dns, provider)) {
              final MonitoringServiceVO dhcpService = _monitorServiceDao.getServiceByName(MonitoringService.Service.Dhcp.toString());
              if (dhcpService != null) {
                  services.add(dhcpService);
              }
          }
  
          if (_networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Lb, provider)) {
              final MonitoringServiceVO lbService = _monitorServiceDao.getServiceByName(MonitoringService.Service.LoadBalancing.toString());
              if (lbService != null) {
                  services.add(lbService);
              }
          }
  
          services.addAll(getDefaultServicesToMonitor(network));
  
          final List<MonitorServiceTO> servicesTO = new ArrayList<>();
          for (final MonitoringServiceVO service : services) {
              final MonitorServiceTO serviceTO = new MonitorServiceTO(service.getService(), service.getProcessName(), service.getServiceName(), service.getServicePath(),
                      service.getServicePidFile(), service.isDefaultService());
              servicesTO.add(serviceTO);
          }
  
          // TODO : This is a hacking fix
          // at VR startup time, information in VirtualMachineProfile may not
          // updated to DB yet,
          // getRouterControlIp() may give wrong IP under basic network mode in
          // VMware environment
          final NicProfile controlNic = getControlNic(profile);
          if (controlNic == null) {
              throw new CloudRuntimeException("VirtualMachine " + profile.getInstanceName() + " doesn't have a control interface");
          }
  
          // As part of aggregate command we don't need to reconfigure if onStart and persist in processed cache. Subsequent updates are not needed.
          SetMonitorServiceCommand command = createMonitorServiceCommand(router, servicesTO, !onStart, false, routerHealthCheckConfig);
          command.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(networkId, router.getId()));
          if (!isMonitoringServicesEnabled) {
              command.setAccessDetail(SetMonitorServiceCommand.ROUTER_MONITORING_ENABLED, Boolean.toString(isMonitoringServicesEnabled));
          }
  
          cmds.addCommand("monitor", command);
      }
  
      protected List<MonitoringServiceVO> getDefaultServicesToMonitor(final NetworkVO network) {
          return _monitorServiceDao.listDefaultServices(true);
      }
  
      protected NicProfile getControlNic(final VirtualMachineProfile profile) {
          NicProfile controlNic = null;
          for (final NicProfile nic : profile.getNics()) {
              if (nic.getTrafficType() == TrafficType.Control && nic.getIPv4Address() != null) {
                  controlNic = nic;
              }
          }
          return controlNic;
      }
  
      protected NicProfile getGuestNic(final VirtualMachineProfile profile) {
          NicProfile guestNic = null;
          for (final NicProfile nic : profile.getNics()) {
              if (nic.getTrafficType() == TrafficType.Guest) {
                  guestNic = nic;
                  break;
              }
          }
          return guestNic;
      }
  
      protected void finalizeSshAndVersionAndNetworkUsageOnStart(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final NicProfile controlNic) {
          final DomainRouterVO vr = _routerDao.findById(profile.getId());
          cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIPv4Address(), 3922));
  
          // Update router template/scripts version
          final GetDomRVersionCmd command = new GetDomRVersionCmd();
          command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIPv4Address());
          command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
          cmds.addCommand("getDomRVersion", command);
  
          // Network usage command to create iptables rules
          final boolean forVpc = vr.getVpcId() != null;
          if (!forVpc) {
              final NicProfile guestNic = getGuestNic(profile);
              if (guestNic != null) {
                  Network network = _networkDao.findById(guestNic.getNetworkId());
                  if (!routedIpv4Manager.isRoutedNetwork(network)) {
                      cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIPv4Address(), router.getHostName(), "create", forVpc));
                  }
              }
          }
      }
  
      protected void finalizeUserDataAndDhcpOnStart(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) {
          if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)
                  || _networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dns, provider)) {
              // Resend dhcp
              logger.debug("Reapplying dhcp entries as a part of domR " + router + " start...");
              _commandSetupHelper.createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId);
          }
  
          if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) {
              // Resend user data
              logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start...");
              _commandSetupHelper.createVmDataCommandForVMs(router, cmds, guestNetworkId);
          }
      }
  
      protected void finalizeNetworkRulesForNetwork(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) {
          logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start");
  
          final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(provider, guestNetworkId);
          final List<FirewallRule> firewallRulesEgress = new ArrayList<>();
          final List<FirewallRule> ipv6firewallRules = new ArrayList<>();
  
          // Fetch firewall Egress rules.
          if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) {
              firewallRulesEgress.addAll(_rulesDao.listByNetworkPurposeTrafficType(guestNetworkId, Purpose.Firewall, FirewallRule.TrafficType.Egress));
              // create egress default rule for VR
              createDefaultEgressFirewallRule(firewallRulesEgress, guestNetworkId);
  
              // add routing ingress firewall rules which do not have public IPs
              firewallRulesEgress.addAll(_rulesDao.listRoutingIngressFirewallRules(guestNetworkId));
  
              // create egress default Ipv6 rules for VR
              createDefaultEgressIpv6FirewallRule(ipv6firewallRules, guestNetworkId);
              ipv6firewallRules.addAll(_rulesDao.listByNetworkPurposeTrafficType(guestNetworkId, Purpose.Ipv6Firewall, FirewallRule.TrafficType.Egress));
              ipv6firewallRules.addAll(_rulesDao.listByNetworkPurposeTrafficType(guestNetworkId, Purpose.Ipv6Firewall, FirewallRule.TrafficType.Ingress));
          }
  
          // Re-apply firewall Egress rules
          logger.debug("Found " + firewallRulesEgress.size() + " firewall Egress rule(s) to apply as a part of domR " + router + " start.");
          if (!firewallRulesEgress.isEmpty()) {
              _commandSetupHelper.createFirewallRulesCommands(firewallRulesEgress, router, cmds, guestNetworkId);
          }
  
          logger.debug(String.format("Found %d Ipv6 firewall rule(s) to apply as a part of domR %s start.", ipv6firewallRules.size(), router));
          if (!ipv6firewallRules.isEmpty()) {
              _commandSetupHelper.createIpv6FirewallRulesCommands(ipv6firewallRules, router, cmds, guestNetworkId);
          }
  
          // Apply BGP peers
          final Network guestNetwork = _networkDao.findById(guestNetworkId);
          if (guestNetwork.getVpcId() != null) {
              final Vpc vpc = _vpcDao.findById(guestNetwork.getVpcId());
              if (routedIpv4Manager.isDynamicRoutedVpc(vpc)) {
                  List<? extends BgpPeer> bgpPeers = bgpService.getBgpPeersForVpc(vpc);
                  _commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, guestNetwork);
              }
          } else {
              if (routedIpv4Manager.isDynamicRoutedNetwork(guestNetwork)) {
                  List<? extends BgpPeer> bgpPeers = bgpService.getBgpPeersForNetwork(guestNetwork);
                  _commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, guestNetwork);
              }
          }
  
          if (publicIps != null && !publicIps.isEmpty()) {
              final List<RemoteAccessVpn> vpns = new ArrayList<>();
              final List<PortForwardingRule> pfRules = new ArrayList<>();
              final List<FirewallRule> staticNatFirewallRules = new ArrayList<>();
              final List<StaticNat> staticNats = new ArrayList<>();
              final List<FirewallRule> firewallRulesIngress = new ArrayList<>();
  
              // Get information about all the rules (StaticNats and
              // StaticNatRules; PFVPN to reapply on domR start)
              for (final PublicIpAddress ip : publicIps) {
                  if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) {
                      pfRules.addAll(_pfRulesDao.listForApplication(ip.getId()));
                  }
                  if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) {
                      staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat));
                  }
                  if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) {
                      firewallRulesIngress.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall));
                  }
  
                  if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) {
                      final RemoteAccessVpn vpn = _vpnDao.findByPublicIpAddress(ip.getId());
                      if (vpn != null) {
                          vpns.add(vpn);
                      }
                  }
  
                  if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) {
                      if (ip.isOneToOneNat()) {
  
                          boolean revoke = false;
                          if (ip.getState() == IpAddress.State.Releasing ) {
                              // for ips got struck in releasing state we need to delete the rule not add.
                              logger.debug("Rule revoke set to true for the ip " + ip.getAddress() +" because it is in releasing state");
                              revoke = true;
                          }
                          final StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), ip.getVmIp(), revoke);
  
                          staticNats.add(staticNat);
                      }
                  }
              }
  
              // Re-apply static nats
              logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start.");
              if (!staticNats.isEmpty()) {
                  _commandSetupHelper.createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId);
              }
  
              // Re-apply firewall Ingress rules
              logger.debug("Found " + firewallRulesIngress.size() + " firewall Ingress rule(s) to apply as a part of domR " + router + " start.");
              if (!firewallRulesIngress.isEmpty()) {
                  _commandSetupHelper.createFirewallRulesCommands(firewallRulesIngress, router, cmds, guestNetworkId);
              }
  
              // Re-apply port forwarding rules
              logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start.");
              if (!pfRules.isEmpty()) {
                  _commandSetupHelper.createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId);
              }
  
              // Re-apply static nat rules
              logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start.");
              if (!staticNatFirewallRules.isEmpty()) {
                  final List<StaticNatRule> staticNatRules = new ArrayList<>();
                  for (final FirewallRule rule : staticNatFirewallRules) {
                      staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false));
                  }
                  _commandSetupHelper.createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId);
              }
  
              // Re-apply vpn rules
              logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start.");
              if (!vpns.isEmpty()) {
                  for (final RemoteAccessVpn vpn : vpns) {
                      _commandSetupHelper.createApplyVpnCommands(true, vpn, router, cmds);
                  }
              }
  
              createApplyLoadBalancingRulesCommands(cmds, router, provider, guestNetworkId);
          }
          // Reapply dhcp and dns configuration.
          if (guestNetwork.getGuestType() == GuestType.Shared && _networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) {
              final Map<Network.Capability, String> dhcpCapabilities = _networkSvc.getNetworkOfferingServiceCapabilities(
                      _networkOfferingDao.findById(_networkDao.findById(guestNetworkId).getNetworkOfferingId()), Service.Dhcp);
              final String supportsMultipleSubnets = dhcpCapabilities.get(Network.Capability.DhcpAccrossMultipleSubnets);
              if (Boolean.parseBoolean(supportsMultipleSubnets)) {
                  final List<NicIpAliasVO> revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.State.revoked);
                  logger.debug("Found {} ip Aliases to revoke on the router as a part of dhcp configuration", revokedIpAliasVOs.size());
                  removeRevokedIpAliasFromDb(revokedIpAliasVOs);
  
                  final List<NicIpAliasVO> aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.State.active);
                  logger.debug("Found {} ip Aliases to apply on the router as a part of dhcp configuration", aliasVOs.size());
                  final List<IpAliasTO> activeIpAliasTOs = new ArrayList<>();
                  for (final NicIpAliasVO aliasVO : aliasVOs) {
                      activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString()));
                  }
                  if (!activeIpAliasTOs.isEmpty()) {
                      _commandSetupHelper.createIpAlias(router, activeIpAliasTOs, guestNetworkId, cmds);
                      _commandSetupHelper.configDnsMasq(router, _networkDao.findById(guestNetworkId), cmds);
                  }
              }
          }
      }
  
      private void createApplyLoadBalancingRulesCommands(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) {
          if (router.getVpcId() != null) {
              return;
          }
          final List<LoadBalancerVO> lbs = _loadBalancerDao.listByNetworkIdAndScheme(guestNetworkId, Scheme.Public);
          final List<LoadBalancingRule> lbRules = new ArrayList<>();
          if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) {
              // Re-apply load balancing rules
              createLoadBalancingRulesList(lbRules, lbs);
          }
  
          logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start.");
          if (!lbRules.isEmpty()) {
              _commandSetupHelper.createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId);
          }
      }
  
      protected void createLoadBalancingRulesList(List<LoadBalancingRule> lbRules, final List<LoadBalancerVO> lbs) {
          for (final LoadBalancerVO lb : lbs) {
              final List<LbDestination> dstList = _lbMgr.getExistingDestinations(lb.getId());
              final List<LbStickinessPolicy> policyList = _lbMgr.getStickinessPolicies(lb.getId());
              final List<LbHealthCheckPolicy> hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId());
              final Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress();
              final LbSslCert sslCert = _lbMgr.getLbSslCert(lb.getId());
              final LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp, sslCert, lb.getLbProtocol());
              lbRules.add(loadBalancing);
          }
      }
  
      private void createDefaultEgressFirewallRule(final List<FirewallRule> rules, final long networkId) {
          final NetworkVO network = _networkDao.findById(networkId);
          final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
          final boolean defaultEgressPolicy = offering.isEgressDefaultPolicy();
  
          // The default on the router is set to Deny all. So, if the default configuration in the offering is set to true (Allow), we change the Egress here
          if (defaultEgressPolicy) {
              final FirewallRule rule = getFirewallRule(network.getCidr(), NetUtils.ALL_IP4_CIDRS, networkId, network, Purpose.Firewall);
  
              rules.add(rule);
          } else {
              logger.debug("Egress policy for the Network {} is already defined as Deny. So, no need to default the rule to Allow. ", network);
          }
      }
  
      private static FirewallRule getFirewallRule(String cidr, String allIp4Cidrs, long networkId, NetworkVO network, Purpose firewall) {
          final List<String> sourceCidr = new ArrayList<>();
          final List<String> destCidr = new ArrayList<>();
  
          sourceCidr.add(cidr);
          destCidr.add(allIp4Cidrs);
  
          return new FirewallRuleVO(null, null, null, null, NetUtils.ALL_PROTO, networkId, network.getAccountId(), network.getDomainId(), firewall, sourceCidr,
                  destCidr, null, null, null, FirewallRule.TrafficType.Egress, FirewallRule.FirewallRuleType.System);
      }
  
      private void createDefaultEgressIpv6FirewallRule(final List<FirewallRule> rules, final long networkId) {
          final NetworkVO network = _networkDao.findById(networkId);
          if(!_networkOfferingDao.isIpv6Supported(network.getNetworkOfferingId())) {
              return;
          }
          // Since not all networks will IPv6 supported, add a system rule for IPv6 networks
          final FirewallRule rule = getFirewallRule(network.getIp6Cidr(), NetUtils.ALL_IP6_CIDRS, networkId, network, Purpose.Ipv6Firewall);
          rules.add(rule);
      }
  
      private void removeRevokedIpAliasFromDb(final List<NicIpAliasVO> revokedIpAliasVOs) {
          for (final NicIpAliasVO ipalias : revokedIpAliasVOs) {
              _nicIpAliasDao.expunge(ipalias.getId());
          }
      }
  
      protected void finalizeIpAssocForNetwork(final Commands cmds, final VirtualRouter router, final Provider provider, final Long guestNetworkId,
              final Map<String, String> vlanMacAddress) {
  
          final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(provider, guestNetworkId);
  
          if (publicIps != null && !publicIps.isEmpty()) {
              logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start.");
              // Re-apply public ip addresses - should come before PF/LB/VPN
              if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) {
                  _commandSetupHelper.createAssociateIPCommands(router, publicIps, cmds, 0);
              }
          }
      }
  
      protected ArrayList<? extends PublicIpAddress> getPublicIpsToApply(final Provider provider, final Long guestNetworkId,
              final com.cloud.network.IpAddress.State... skipInStates) {
  
          final List<? extends IpAddress> userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null);
  
          final List<PublicIp> allPublicIps = new ArrayList<>();
          if (userIps != null && !userIps.isEmpty()) {
              boolean addIp = true;
              for (final IpAddress userIp : userIps) {
                  if (skipInStates != null) {
                      for (final IpAddress.State stateToSkip : skipInStates) {
                          if (userIp.getState() == stateToSkip) {
                              logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState());
                              addIp = false;
                              break;
                          }
                      }
                  }
  
                  if (addIp) {
                      final IPAddressVO ipVO = _ipAddressDao.findById(userIp.getId());
                      final PublicIp publicIp = PublicIp.createFromAddrAndVlan(ipVO, _vlanDao.findById(userIp.getVlanId()));
                      allPublicIps.add(publicIp);
                  }
              }
          }
  
          // Get public Ips that should be handled by router
          final Network network = _networkDao.findById(guestNetworkId);
          final Map<PublicIpAddress, Set<Service>> ipToServices = _networkModel.getIpToServices(allPublicIps, false, true);
          final Map<Provider, ArrayList<PublicIpAddress>> providerToIpList = _networkModel.getProviderToIpList(network, ipToServices);
          // Only cover virtual router for now, if ELB use it this need to be
          // modified
  
          return providerToIpList.get(provider);
      }
  
      @Override
      public boolean finalizeStart(final VirtualMachineProfile profile, final long hostId, final Commands cmds, final ReservationContext context) {
          final DomainRouterVO router = _routerDao.findById(profile.getId());
  
          // process all the answers
          for (final Answer answer : cmds.getAnswers()) {
              // handle any command failures
              if (!answer.getResult()) {
                  final String cmdClassName = answer.getClass().getCanonicalName().replace("Answer", "Command");
                  final String errorMessage = "Command: " + cmdClassName + " failed while starting virtual router";
                  final String errorDetails = "Details: " + answer.getDetails() + " " + answer;
                  // add alerts for the failed commands
                  _alertMgr.sendAlert(AlertService.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails);
                  logger.error(answer.getDetails());
                  logger.warn(errorMessage);
                  // Stop the router if any of the commands failed
                  return false;
              }
          }
  
          // at this point, all the router command are successful.
          boolean result = true;
          // Get guest networks info
          final List<Network> guestNetworks = new ArrayList<>();
  
          final GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer) cmds.getAnswer("getDomRVersion");
          router.setTemplateVersion(versionAnswer.getTemplateVersion());
          router.setScriptsVersion(versionAnswer.getScriptsVersion());
          String codeVersion = mgr.getVersion();
          if (StringUtils.isNotEmpty(codeVersion)) {
              codeVersion = CloudStackVersion.parse(codeVersion).toString();
          }
          router.setSoftwareVersion(codeVersion);
          _routerDao.persist(router, guestNetworks);
  
          final List<? extends Nic> routerNics = _nicDao.listByVmId(profile.getId());
          for (final Nic nic : routerNics) {
              final Network network = _networkModel.getNetwork(nic.getNetworkId());
  
              final DataCenterVO dcVO = _dcDao.findById(network.getDataCenterId());
  
              if (network.getTrafficType() == TrafficType.Guest) {
                  guestNetworks.add(network);
                  if (nic.getBroadcastUri().getScheme().equals("pvlan")) {
                      final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 0, false, "pvlan-nic");
  
                      final NetworkTopology networkTopology = _networkTopologyContext.retrieveNetworkTopology(dcVO);
                      try {
                          result = networkTopology.setupDhcpForPvlan(true, router, router.getHostId(), nicProfile);
                      } catch (final ResourceUnavailableException e) {
                          logger.debug("ERROR in finalizeStart: ", e);
                      }
                  }
              }
          }
  
          if (result) {
              for (Network guestNetwork : guestNetworks) {
                  _routerDao.addRouterToGuestNetwork(router, guestNetwork);
              }
          }
  
          return result;
      }
  
      @Override
      public void finalizeStop(final VirtualMachineProfile profile, final Answer answer) {
          if (answer != null) {
              final VirtualMachine vm = profile.getVirtualMachine();
              final DomainRouterVO domR = _routerDao.findById(vm.getId());
              processStopOrRebootAnswer(domR, answer);
              if (Boolean.TRUE.equals(RemoveControlIpOnStop.valueIn(profile.getVirtualMachine().getDataCenterId()))) {
                  removeNics(vm, domR);
              }
          }
      }
  
      @Override
      public void finalizeExpunge(final VirtualMachine vm) {
          if (Boolean.FALSE.equals(RemoveControlIpOnStop.valueIn(vm.getDataCenterId()))) {
              final DomainRouterVO domR = _routerDao.findById(vm.getId());
              logger.info(String.format("removing nics for VR [%s]", vm));
              removeNics(vm, domR);
          }
      }
  
      private void removeNics(VirtualMachine vm, DomainRouterVO domR) {
          final List<? extends Nic> routerNics = _nicDao.listByVmId(vm.getId());
          final DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
  
          for (final Nic nic : routerNics) {
              final Network network = _networkModel.getNetwork(nic.getNetworkId());
  
              removeDhcpRulesForPvLan(domR, nic, network, dcVO);
          }
      }
  
      private void removeDhcpRulesForPvLan(DomainRouterVO domR, Nic nic, Network network, DataCenterVO dcVO) {
          if (network.getTrafficType() == TrafficType.Guest && nic.getBroadcastUri() != null && nic.getBroadcastUri().getScheme().equals("pvlan")) {
              final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 0, false, "pvlan-nic");
  
              final NetworkTopology networkTopology = _networkTopologyContext.retrieveNetworkTopology(dcVO);
              try {
                  networkTopology.setupDhcpForPvlan(false, domR, domR.getHostId(), nicProfile);
              } catch (final ResourceUnavailableException e) {
                  logger.debug("ERROR in finalizeStop: ", e);
              }
          }
      }
  
      @Override
      public boolean startRemoteAccessVpn(final Network network, final RemoteAccessVpn vpn, final List<? extends VirtualRouter> routers) throws ResourceUnavailableException {
          if (routers == null || routers.isEmpty()) {
              logger.warn("Failed to start remote access VPN: no router found for account and zone");
              throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", DataCenter.class, network.getDataCenterId());
          }
  
          for (final VirtualRouter router : routers) {
              if (router.getState() != VirtualMachine.State.Running) {
                  logger.warn("Failed to start remote access VPN: router not in right state " + router.getState());
                  throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " + router.getState(), DataCenter.class,
                          network.getDataCenterId());
              }
  
              final Commands cmds = new Commands(Command.OnError.Stop);
              _commandSetupHelper.createApplyVpnCommands(true, vpn, router, cmds);
  
              if (!_nwHelper.sendCommandsToRouter(router, cmds)) {
                  throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId());
              }
  
              Answer answer = cmds.getAnswer("users");
              if (answer == null) {
                  DataCenterVO zone = _dcDao.findById(router.getDataCenterId());
                  Account account = _accountMgr.getAccount(vpn.getAccountId());
                  logger.error("Unable to start vpn {} : unable add users to vpn in zone {} for account {} on domR: {} due to null answer", vpn, zone, account, router.getInstanceName());
                  throw new ResourceUnavailableException(String.format("Unable to start vpn %s in zone %s for account %s on domR: %s due to null answer", vpn, zone, account, router.getInstanceName()), DataCenter.class, router.getDataCenterId());
              }
              if (!answer.getResult()) {
                  DataCenterVO zone = _dcDao.findById(router.getDataCenterId());
                  Account account = _accountMgr.getAccount(vpn.getAccountId());
                  logger.error("Unable to start vpn {} : unable add users to vpn in zone {} for account {} on domR: {} due to {}", vpn, zone, account, router.getInstanceName(), answer.getDetails());
                  throw new ResourceUnavailableException(String.format("Unable to start vpn %s : Unable to add users to vpn in zone %s for account %s on domR: %s due to %s", vpn, zone, account, router.getInstanceName(), answer.getDetails()), DataCenter.class, router.getDataCenterId());
              }
              answer = cmds.getAnswer("startVpn");
              if (!answer.getResult()) {
                  DataCenterVO zone = _dcDao.findById(router.getDataCenterId());
                  Account account = _accountMgr.getAccount(vpn.getAccountId());
  
                  logger.error("Unable to start vpn {}  in zone {} for account {} on domR: {} due to {}", vpn, zone, account, router.getInstanceName(), answer.getDetails());
                  throw new ResourceUnavailableException(String.format("Unable to start vpn %s  in zone %s for account %s on domR: %s due to %s", vpn, zone, account, router.getInstanceName(), answer.getDetails()), DataCenter.class, router.getDataCenterId());
              }
  
          }
          return true;
      }
  
      @Override
      public boolean deleteRemoteAccessVpn(final Network network, final RemoteAccessVpn vpn, final List<? extends VirtualRouter> routers) throws ResourceUnavailableException {
          if (routers == null || routers.isEmpty()) {
              logger.warn("Failed to delete remote access VPN: no router found for account and zone");
              throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId());
          }
  
          boolean result = true;
          for (final VirtualRouter router : routers) {
              if (router.getState() == VirtualMachine.State.Running) {
                  final Commands cmds = new Commands(Command.OnError.Continue);
                  _commandSetupHelper.createApplyVpnCommands(false, vpn, router, cmds);
                  result = result && _nwHelper.sendCommandsToRouter(router, cmds);
              } else if (router.getState() == VirtualMachine.State.Stopped) {
                  logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it");
              } else {
                  logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState());
                  throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + router.getState(), DataCenter.class,
                          network.getDataCenterId());
              }
          }
  
          return result;
      }
  
      @Override
      public DomainRouterVO stop(final VirtualRouter router, final boolean forced, final User user, final Account caller) throws ConcurrentOperationException,
      ResourceUnavailableException {
          logger.debug("Stopping router " + router);
          try {
              _itMgr.advanceStop(router.getUuid(), forced);
              return _routerDao.findById(router.getId());
          } catch (final OperationTimedoutException e) {
              throw new CloudRuntimeException("Unable to stop " + router, e);
          }
      }
  
      @Override
      public boolean removeDhcpSupportForSubnet(final Network network, final List<DomainRouterVO> routers) throws ResourceUnavailableException {
          if (routers == null || routers.isEmpty()) {
              logger.warn("Failed to add/remove VPN users: no router found for account and zone");
              throw new ResourceUnavailableException(String.format("Unable to assign ip addresses, domR doesn't exist for network %s", network), DataCenter.class, network.getDataCenterId());
          }
  
          for (final DomainRouterVO router : routers) {
              if (router.getState() != VirtualMachine.State.Running) {
                  logger.warn("Failed to add/remove VPN users: router not in running state");
                  throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + router.getState(), DataCenter.class,
                          network.getDataCenterId());
              }
  
              final Commands cmds = new Commands(Command.OnError.Continue);
              final List<NicIpAliasVO> revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.State.revoked);
              logger.debug("Found" + revokedIpAliasVOs.size() + "ip Aliases to revoke on the router as a part of dhcp configuration");
              final List<IpAliasTO> revokedIpAliasTOs = new ArrayList<>();
              for (final NicIpAliasVO revokedAliasVO : revokedIpAliasVOs) {
                  revokedIpAliasTOs.add(new IpAliasTO(revokedAliasVO.getIp4Address(), revokedAliasVO.getNetmask(), revokedAliasVO.getAliasCount().toString()));
              }
              final List<NicIpAliasVO> aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.State.active);
              logger.debug("Found" + aliasVOs.size() + "ip Aliases to apply on the router as a part of dhcp configuration");
              final List<IpAliasTO> activeIpAliasTOs = new ArrayList<>();
              for (final NicIpAliasVO aliasVO : aliasVOs) {
                  activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString()));
              }
              _commandSetupHelper.createDeleteIpAliasCommand(router, revokedIpAliasTOs, activeIpAliasTOs, network.getId(), cmds);
              _commandSetupHelper.configDnsMasq(router, network, cmds);
              final boolean result = _nwHelper.sendCommandsToRouter(router, cmds);
              if (result) {
                  Transaction.execute(new TransactionCallbackNoReturn() {
                      @Override
                      public void doInTransactionWithoutResult(final TransactionStatus status) {
                          for (final NicIpAliasVO revokedAliasVO : revokedIpAliasVOs) {
                              _nicIpAliasDao.expunge(revokedAliasVO.getId());
                          }
                      }
                  });
                  return true;
              }
          }
          return false;
      }
  
      @Override
      @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true)
      public VirtualRouter startRouter(final long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException {
          return startRouter(id, true);
      }
  
      @Override
      public VirtualRouter startRouter(final long routerId, final boolean reprogramNetwork) throws ResourceUnavailableException, InsufficientCapacityException,
      ConcurrentOperationException {
          final Account caller = CallContext.current().getCallingAccount();
          final User callerUser = _accountMgr.getActiveUser(CallContext.current().getCallingUserId());
  
          // verify parameters
          DomainRouterVO router = _routerDao.findById(routerId);
          //clean up the update_state feild
          if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_FAILED){
              router.setUpdateState(null);
              _routerDao.update(router.getId(),router);
          }
          _accountMgr.checkAccess(caller, null, true, router);
  
          final Account owner = _accountMgr.getAccount(router.getAccountId());
  
          // Check if all networks are implemented for the domR; if not -
          // implement them
          final DataCenter dc = _dcDao.findById(router.getDataCenterId());
          HostPodVO pod = null;
          if (router.getPodIdToDeployIn() != null) {
              pod = _podDao.findById(router.getPodIdToDeployIn());
          }
          final DeployDestination dest = new DeployDestination(dc, pod, null, null);
  
          final ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner);
  
          final List<NicVO> nics = _nicDao.listByVmId(routerId);
  
          for (final NicVO nic : nics) {
              if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) {
                  NetworkVO network = _networkDao.findById(nic.getNetworkId());
                  logger.warn("Failed to start network {} as a part of domR start", network);
                  throw new CloudRuntimeException(String.format("Failed to start network %s as a part of domR start", network));
              }
          }
  
          // After start network, check if it's already running
          router = _routerDao.findById(routerId);
          if (router.getState() == VirtualMachine.State.Running) {
              return router;
          }
  
          final UserVO user = _userDao.findById(CallContext.current().getCallingUserId());
          final Map<Param, Object> params = new HashMap<>();
          if (reprogramNetwork) {
              params.put(Param.ReProgramGuestNetworks, true);
          } else {
              params.put(Param.ReProgramGuestNetworks, false);
          }
          final VirtualRouter virtualRouter = _nwHelper.startVirtualRouter(router, user, caller, params);
          if (virtualRouter == null) {
              throw new CloudRuntimeException(String.format("Failed to start router %s", router));
          }
          return virtualRouter;
      }
  
      @Override
      @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "restarting router VM for HA", async = true)
      public void startRouterForHA(VirtualMachine vm, Map<Param, Object> params, DeploymentPlanner planner)
              throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
              OperationTimedoutException {
          _itMgr.advanceStart(vm.getUuid(), params, planner);
      }
  
      @Override
      public List<VirtualRouter> getRoutersForNetwork(final long networkId) {
          final List<DomainRouterVO> routers = _routerDao.findByNetwork(networkId);
          final List<VirtualRouter> vrs = new ArrayList<>(routers.size());
          vrs.addAll(routers);
          return vrs;
      }
  
      @Override
      public String getDnsBasicZoneUpdate() {
          return _dnsBasicZoneUpdates;
      }
  
      @Override
      public int getTimeout() {
          return -1;
      }
  
      @Override
      public boolean isRecurring() {
          return false;
      }
  
      @Override
      public boolean processAnswers(final long agentId, final long seq, final Answer[] answers) {
          return false;
      }
  
      @Override
      public boolean processCommands(final long agentId, final long seq, final Command[] commands) {
          return false;
      }
  
      @Override
      public void processHostAdded(long hostId) {
      }
  
      @Override
      public void processConnect(final Host host, final StartupCommand cmd, final boolean forRebalance) throws ConnectionException {
          final List<DomainRouterVO> routers = _routerDao.listIsolatedByHostId(host.getId());
          for (DomainRouterVO router : routers) {
              if (router.isStopPending()) {
                  logger.info("Stopping router " + router.getInstanceName() + " due to stop pending flag found!");
                  final VirtualMachine.State state = router.getState();
                  if (state != VirtualMachine.State.Stopped && state != VirtualMachine.State.Destroyed) {
                      try {
                          stopRouter(router.getId(), false);
                      } catch (final ResourceUnavailableException | ConcurrentOperationException e) {
                          logger.warn("Fail to stop router " + router.getInstanceName(), e);
                          throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName());
                      }
                  }
                  router.setStopPending(false);
                  _routerDao.persist(router);
              }
          }
      }
  
      @Override
      public AgentControlAnswer processControlCommand(final long agentId, final AgentControlCommand cmd) {
          return null;
      }
  
      @Override
      public boolean processDisconnect(final long agentId, final Status state) {
          return false;
      }
  
      @Override
      public void processHostAboutToBeRemoved(long hostId) {
      }
  
      @Override
      public void processHostRemoved(long hostId, long clusterId) {
      }
  
      @Override
      public boolean processTimeout(final long agentId, final long seq) {
          return false;
      }
  
      @Override
      public void prepareStop(final VirtualMachineProfile profile) {
          // Collect network usage before stopping Vm
  
          final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId());
          collectNetworkStatistics(router, null);
      }
  
      @Override
      public <T extends VirtualRouter> void collectNetworkStatistics(final T router, final Nic nic) {
          if (router == null) {
              return;
          }
  
          final String privateIP = router.getPrivateIpAddress();
  
          if (privateIP != null) {
              final boolean forVpc = router.getVpcId() != null;
              List<Nic> routerNics = new ArrayList<>();
              if (nic != null) {
                  routerNics.add(nic);
              } else {
                  routerNics.addAll(_nicDao.listByVmId(router.getId()));
              }
              for (final Nic routerNic : routerNics) {
                  final Network network = _networkModel.getNetwork(routerNic.getNetworkId());
                  // Send network usage command for public nic in VPC VR
                  // Send network usage command for isolated guest nic of non VPC
                  // VR
  
                  //[TODO] Avoiding the NPE now, but I have to find out what is going on with the network. - Wilder Rodrigues
                  if (network == null) {
                      logger.error("Could not find a network with ID => " + routerNic.getNetworkId() + ". It might be a problem!");
                      continue;
                  }
                  if (routedIpv4Manager.isRoutedNetwork(network)) {
                      continue;
                  }
                  if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest
                          && network.getGuestType() == Network.GuestType.Isolated) {
                      final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address());
                      final String routerType = router.getType().toString();
                      final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(),
                              forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType);
                      NetworkUsageAnswer answer;
                      try {
                          answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd);
                      } catch (final Exception e) {
                          logger.warn("Error while collecting network stats from router: {} from host: {}", router, router.getHostId(), e);
                          continue;
                      }
  
                      if (answer != null) {
                          if (!answer.getResult()) {
                              logger.warn("Error while collecting network stats from router: {} from host: {}; details: {}", router, router.getHostId(), answer.getDetails());
                              continue;
                          }
                          try {
                              if (answer.getBytesReceived() == 0 && answer.getBytesSent() == 0) {
                                  logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics");
                                  continue;
                              }
  
                              final NetworkUsageAnswer answerFinal = answer;
                              Transaction.execute(new TransactionCallbackNoReturn() {
                                  @Override
                                  public void doInTransactionWithoutResult(final TransactionStatus status) {
                                      final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(),
                                              forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType);
                                      if (stats == null) {
                                          logger.warn("unable to find stats for account: {}", () -> _accountMgr.getAccount(router.getAccountId()));
                                          return;
                                      }
  
                                      if (previousStats != null
                                              && (previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived() || previousStats.getCurrentBytesSent() != stats
                                              .getCurrentBytesSent())) {
                                          logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: "
                                                  + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent());
                                          return;
                                      }
  
                                      if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) {
                                          logger.debug("Received # of bytes that's less than the last one. Assuming something went wrong and persisting it. Router: {} Reported: {} Stored: {}"
                                                      , answerFinal.getRouterName()
                                                      , toHumanReadableSize(answerFinal.getBytesReceived())
                                                      , toHumanReadableSize(stats.getCurrentBytesReceived()));
                                          stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived());
                                      }
                                      stats.setCurrentBytesReceived(answerFinal.getBytesReceived());
                                      if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) {
                                          logger.debug("Received # of bytes that's less than the last one. Assuming something went wrong and persisting it. Router: {} Reported: {} Stored: {}"
                                                  , answerFinal.getRouterName()
                                                  , toHumanReadableSize(answerFinal.getBytesReceived())
                                                  , toHumanReadableSize(stats.getCurrentBytesReceived()));
                                          stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent());
                                      }
                                      stats.setCurrentBytesSent(answerFinal.getBytesSent());
                                      if (!_dailyOrHourly) {
                                          // update agg bytes
                                          stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent());
                                          stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived());
                                      }
                                      _userStatsDao.update(stats.getId(), stats);
                                  }
                              });
                          } catch (final Exception e) {
                              logger.warn("Unable to update user statistics for account: {} Rx: {}; Tx: {}",
                                      _accountMgr.getAccount(router.getAccountId()), toHumanReadableSize(answer.getBytesReceived()), toHumanReadableSize(answer.getBytesSent()));
                          }
                      }
                  }
              }
          }
      }
  
      @Override
      public void finalizeUnmanage(VirtualMachine vm) {
      }
  
      @Override
      public VirtualRouter findRouter(final long routerId) {
          return _routerDao.findById(routerId);
      }
  
      @Override
      public List<Long> upgradeRouterTemplate(final UpgradeRouterTemplateCmd cmd) {
  
          List<DomainRouterVO> routers = new ArrayList<>();
          int params = 0;
  
          final Long routerId = cmd.getId();
          if (routerId != null) {
              params++;
              final DomainRouterVO router = _routerDao.findById(routerId);
              if (router != null) {
                  routers.add(router);
              }
          }
  
          final Long domainId = cmd.getDomainId();
          if (domainId != null) {
              final String accountName = cmd.getAccount();
              // List by account, if account Name is specified along with domainId
              if (accountName != null) {
                  final Account account = _accountMgr.getActiveAccountByName(accountName, domainId);
                  if (account == null) {
                      throw new InvalidParameterValueException("Account :" + accountName + " does not exist in domain: " + domainId);
                  }
                  routers = _routerDao.listRunningByAccountId(account.getId());
              } else {
                  // List by domainId, account name not specified
                  routers = _routerDao.listRunningByDomain(domainId);
              }
              params++;
          }
  
          final Long clusterId = cmd.getClusterId();
          if (clusterId != null) {
              params++;
              routers = _routerDao.listRunningByClusterId(clusterId);
          }
  
          final Long podId = cmd.getPodId();
          if (podId != null) {
              params++;
              routers = _routerDao.listRunningByPodId(podId);
          }
  
          final Long zoneId = cmd.getZoneId();
          if (zoneId != null) {
              params++;
              routers = _routerDao.listRunningByDataCenter(zoneId);
          }
  
          if (params > 1) {
              throw new InvalidParameterValueException("Multiple parameters not supported. Specify only one among routerId/zoneId/podId/clusterId/accountId/domainId");
          }
  
          if (routers != null) {
              return rebootRouters(routers);
          }
  
          return null;
      }
  
      private List<Long> rebootRouters(final List<DomainRouterVO> routers) {
          final List<Long> jobIds = new ArrayList<>();
          for (final DomainRouterVO router : routers) {
              if (!_nwHelper.checkRouterTemplateVersion(router)) {
                  logger.debug("Upgrading template for router: {}", router);
                  final Map<String, String> params = new HashMap<>();
                  params.put("ctxUserId", "1");
                  params.put("ctxAccountId", "" + router.getAccountId());
  
                  final RebootRouterCmd cmd = new RebootRouterCmd();
                  ComponentContext.inject(cmd);
                  params.put("id", "" + router.getId());
                  params.put("ctxStartEventId", "1");
                  final AsyncJobVO job = new AsyncJobVO("", User.UID_SYSTEM, router.getAccountId(), RebootRouterCmd.class.getName(), ApiGsonHelper.getBuilder().create().toJson(params),
                          router.getId(), cmd.getApiResourceType() != null ? cmd.getApiResourceType().toString() : null, null);
                  job.setDispatcher(_asyncDispatcher.getName());
                  final long jobId = _asyncMgr.submitAsyncJob(job);
                  jobIds.add(jobId);
              } else {
                  logger.debug("Router: {} is already at the latest version. No upgrade required", router);
              }
          }
          return jobIds;
      }
  
      @Override
      public String getConfigComponentName() {
          return VirtualNetworkApplianceManagerImpl.class.getSimpleName();
      }
  
      @Override
      public ConfigKey<?>[] getConfigKeys() {
          return new ConfigKey<?>[] {
                  RouterTemplateKvm,
                  RouterTemplateVmware,
                  RouterTemplateHyperV,
                  RouterTemplateLxc,
                  RouterTemplateOvm3,
                  UseExternalDnsServers,
                  RouterVersionCheckEnabled,
                  SetServiceMonitor,
                  VirtualRouterServiceOffering,
                  RouterAlertsCheckInterval,
                  RouterHealthChecksEnabled,
                  RouterHealthChecksBasicInterval,
                  RouterHealthChecksAdvancedInterval,
                  RouterHealthChecksConfigRefreshInterval,
                  RouterHealthChecksResultFetchInterval,
                  RouterHealthChecksFailuresToRecreateVr,
                  RouterHealthChecksToExclude,
                  RouterHealthChecksFreeDiskSpaceThreshold,
                  RouterHealthChecksMaxCpuUsageThreshold,
                  RouterHealthChecksMaxMemoryUsageThreshold,
                  ExposeDnsAndBootpServer,
                  RouterLogrotateFrequency,
                  RemoveControlIpOnStop,
                  VirtualRouterUserData
          };
      }
  
      @Override
      public boolean preStateTransitionEvent(final VirtualMachine.State oldState, final VirtualMachine.Event event, final VirtualMachine.State newState, final VirtualMachine vo, final boolean status,
              final Object opaque) {
          return true;
      }
  
      @Override
      public boolean postStateTransitionEvent(final StateMachine2.Transition<VirtualMachine.State, VirtualMachine.Event> transition, final VirtualMachine vo, final boolean status, final Object opaque) {
          final VirtualMachine.State newState = transition.getToState();
          final VirtualMachine.Event event = transition.getEvent();
          if (vo.getType() == VirtualMachine.Type.DomainRouter &&
                  event == VirtualMachine.Event.FollowAgentPowerOnReport &&
                  newState == VirtualMachine.State.Running &&
                  isOutOfBandMigrated(opaque)) {
              logger.debug("Virtual router " + vo.getInstanceName() + " is powered-on out-of-band");
          }
  
          return true;
      }
  
      private boolean isOutOfBandMigrated(final Object opaque) {
          // opaque -> <hostId, powerHostId>
          if (opaque instanceof Pair<?, ?>) {
              final Pair<?, ?> pair = (Pair<?, ?>)opaque;
              final Object first = pair.first();
              final Object second = pair.second();
              // powerHostId cannot be null in case of out-of-band VM movement
              if (second instanceof Long) {
                  final Long powerHostId = (Long)second;
                  Long hostId = null;
                  if (first instanceof Long) {
                      hostId = (Long)first;
                  }
                  // The following scenarios are due to out-of-band VM movement
                  // 1. If VM is in stopped state in CS due to 'PowerMissing' report from old host (hostId is null) and then there is a 'PowerOn' report from new host
                  // 2. If VM is in running state in CS and there is a 'PowerOn' report from new host
                  return hostId == null || hostId.longValue() != powerHostId.longValue();
              }
          }
          return false;
      }
  
      protected boolean aggregationExecution(final AggregationControlCommand.Action action, final Network network, final List<DomainRouterVO> routers)
              throws ResourceUnavailableException {
  
          int errors = 0;
  
          for (final DomainRouterVO router : routers) {
  
              final String routerControlIp = _routerControlHelper.getRouterControlIp(router.getId());
              final String routerIpInNetwork = _routerControlHelper.getRouterIpInNetwork(network.getId(), router.getId());
  
              if (routerIpInNetwork == null) {
                  // Nic hasn't been created in this router yet. Try to configure the next one.
                  logger.warn("The Network is not configured in the router " + router.getHostName() + " yet. Try the next router!");
                  errors++;
                  continue;
              }
  
              final AggregationControlCommand cmd = new AggregationControlCommand(action, router.getInstanceName(), routerControlIp, routerIpInNetwork);
              final Commands cmds = new Commands(cmd);
              if (!_nwHelper.sendCommandsToRouter(router, cmds)) {
                  return false;
              }
          }
          if (errors == routers.size()) {
              logger.error("aggregationExecution() on " + getClass().getName() + " failed! Network is not configured in any router.");
              return false;
          }
          return true;
      }
  
      @Override
      public boolean prepareAggregatedExecution(final Network network, final List<DomainRouterVO> routers) throws ResourceUnavailableException {
          return aggregationExecution(Action.Start, network, routers);
      }
  
      @Override
      public boolean completeAggregatedExecution(final Network network, final List<DomainRouterVO> routers) throws ResourceUnavailableException {
          return aggregationExecution(Action.Finish, network, routers);
      }
  }
  