content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.network.lb;
  
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.Executors;
  import java.util.concurrent.ScheduledExecutorService;
  import java.util.concurrent.TimeUnit;
  
  import javax.inject.Inject;
  import javax.naming.ConfigurationException;
  
  import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
  import org.apache.cloudstack.config.ApiServiceConfiguration;
  import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
  import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
  import org.apache.cloudstack.managed.context.ManagedContextRunnable;
  import org.apache.cloudstack.userdata.UserDataManager;
  import org.apache.commons.lang3.StringUtils;
  import org.springframework.stereotype.Component;
  
  import com.cloud.agent.AgentManager;
  import com.cloud.agent.api.Answer;
  import com.cloud.agent.api.Command;
  import com.cloud.agent.api.check.CheckSshAnswer;
  import com.cloud.agent.api.check.CheckSshCommand;
  import com.cloud.agent.api.routing.LoadBalancerConfigCommand;
  import com.cloud.agent.api.routing.NetworkElementCommand;
  import com.cloud.agent.api.to.LoadBalancerTO;
  import com.cloud.agent.manager.Commands;
  import com.cloud.configuration.Config;
  import com.cloud.dc.DataCenter;
  import com.cloud.dc.DataCenter.NetworkType;
  import com.cloud.dc.DataCenterVO;
  import com.cloud.dc.dao.ClusterDao;
  import com.cloud.dc.dao.DataCenterDao;
  import com.cloud.dc.dao.HostPodDao;
  import com.cloud.deploy.DeployDestination;
  import com.cloud.exception.AgentUnavailableException;
  import com.cloud.exception.ConcurrentOperationException;
  import com.cloud.exception.InsufficientAddressCapacityException;
  import com.cloud.exception.NetworkRuleConflictException;
  import com.cloud.exception.OperationTimedoutException;
  import com.cloud.exception.ResourceUnavailableException;
  import com.cloud.hypervisor.Hypervisor.HypervisorType;
  import com.cloud.network.ElasticLbVmMapVO;
  import com.cloud.network.Network;
  import com.cloud.network.NetworkModel;
  import com.cloud.network.Networks.TrafficType;
  import com.cloud.network.dao.LoadBalancerDao;
  import com.cloud.network.dao.LoadBalancerVO;
  import com.cloud.network.dao.NetworkDao;
  import com.cloud.network.dao.NetworkVO;
  import com.cloud.network.lb.LoadBalancingRule.LbDestination;
  import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy;
  import com.cloud.network.lb.LoadBalancingRule.LbSslCert;
  import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy;
  import com.cloud.network.lb.dao.ElasticLbVmMapDao;
  import com.cloud.network.router.VirtualRouter;
  import com.cloud.network.rules.FirewallRule;
  import com.cloud.network.rules.LoadBalancer;
  import com.cloud.offering.NetworkOffering;
  import com.cloud.offering.ServiceOffering;
  import com.cloud.offerings.dao.NetworkOfferingDao;
  import com.cloud.service.ServiceOfferingVO;
  import com.cloud.service.dao.ServiceOfferingDao;
  import com.cloud.storage.Storage;
  import com.cloud.user.Account;
  import com.cloud.user.AccountService;
  import com.cloud.utils.NumbersUtil;
  import com.cloud.utils.component.ManagerBase;
  import com.cloud.utils.concurrency.NamedThreadFactory;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.net.Ip;
  import com.cloud.vm.DomainRouterVO;
  import com.cloud.vm.NicProfile;
  import com.cloud.vm.ReservationContext;
  import com.cloud.vm.VirtualMachine;
  import com.cloud.vm.VirtualMachine.State;
  import com.cloud.vm.VirtualMachineGuru;
  import com.cloud.vm.VirtualMachineManager;
  import com.cloud.vm.VirtualMachineProfile;
  import com.cloud.vm.dao.DomainRouterDao;
  import com.cloud.vm.dao.NicDao;
  
  import static com.cloud.network.router.VirtualNetworkApplianceManager.VirtualRouterUserData;
  import static com.cloud.vm.VirtualMachineManager.SystemVmEnableUserData;
  
  @Component
  public class ElasticLoadBalancerManagerImpl extends ManagerBase implements ElasticLoadBalancerManager, VirtualMachineGuru {
  
      @Inject
      private AgentManager _agentMgr;
      @Inject
      private NetworkModel _networkModel;
      @Inject
      private LoadBalancingRulesManager _lbMgr;
      @Inject
      private final DomainRouterDao _routerDao = null;
      @Inject
      protected HostPodDao _podDao = null;
      @Inject
      protected ClusterDao _clusterDao;
      @Inject
      private final DataCenterDao _dcDao = null;
      @Inject
      protected NetworkDao _networkDao;
      @Inject
      protected NetworkOfferingDao _networkOfferingDao;
      @Inject
      private VirtualMachineManager _itMgr;
      @Inject
      private ConfigurationDao _configDao;
      @Inject
      private final ServiceOfferingDao _serviceOfferingDao = null;
      @Inject
      private AccountService _accountService;
      @Inject
      private LoadBalancerDao _lbDao;
      @Inject
      private ElasticLbVmMapDao _elbVmMapDao;
      @Inject
      private NicDao _nicDao;
      @Inject
      private UserDataManager userDataManager;
  
      String _instance;
  
      static final private String SystemVmType = "elbvm";
  
      boolean _enabled;
      TrafficType _frontendTrafficType = TrafficType.Guest;
  
      Account _systemAcct;
      ScheduledExecutorService _gcThreadPool;
      String _mgmtCidr;
  
      Set<Long> _gcCandidateElbVmIds = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
  
      int _elasticLbVmRamSize;
      int _elasticLbvmCpuMHz;
      int _elasticLbvmNumCpu;
  
      private LoadBalanceRuleHandler loadBalanceRuleHandler;
  
      private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds) throws AgentUnavailableException {
          Answer[] answers = null;
          try {
              answers = _agentMgr.send(elbVm.getHostId(), cmds);
          } catch (OperationTimedoutException e) {
              logger.warn("ELB: Timed Out", e);
              throw new AgentUnavailableException("Unable to send commands to virtual elbVm ", elbVm.getHostId(), e);
          }
  
          if (answers == null) {
              return false;
          }
  
          if (answers.length != cmds.size()) {
              return false;
          }
  
          // FIXME: Have to return state for individual command in the future
          if (answers.length > 0) {
              Answer ans = answers[0];
              return ans.getResult();
          }
          return true;
      }
  
      private void createApplyLoadBalancingRulesCommands(List<LoadBalancingRule> rules, DomainRouterVO elbVm, Commands cmds, long guestNetworkId) {
  
          /* XXX: cert */
          LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()];
          int i = 0;
          for (LoadBalancingRule rule : rules) {
              boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke));
              String protocol = rule.getProtocol();
              String algorithm = rule.getAlgorithm();
  
              String elbIp = rule.getSourceIp().addr();
              int srcPort = rule.getSourcePortStart();
              String uuid = rule.getUuid();
              List<LbDestination> destinations = rule.getDestinations();
              LoadBalancerTO lb = new LoadBalancerTO(uuid, elbIp, srcPort, protocol, algorithm, revoked, false, false, destinations);
              lbs[i++] = lb;
          }
  
          NetworkOffering offering = _networkOfferingDao.findById(guestNetworkId);
          String maxconn = null;
          if (offering.getConcurrentConnections() == null) {
              maxconn = NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value().toString();
          } else {
              maxconn = offering.getConcurrentConnections().toString();
          }
          LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, elbVm.getPublicIpAddress(), _nicDao.getIpAddress(guestNetworkId, elbVm.getId()),
                  elbVm.getPrivateIpAddress(), null, null, maxconn, offering.isKeepAliveEnabled());
          cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, elbVm.getPrivateIpAddress());
          cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, elbVm.getInstanceName());
          //FIXME: why are we setting attributes directly? Ick!! There should be accessors and
          //the constructor should set defaults.
          cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key());
          cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key());
          cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key());
          cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key());
  
          cmds.addCommand(cmd);
  
      }
  
      protected boolean applyLBRules(DomainRouterVO elbVm, List<LoadBalancingRule> rules, long guestNetworkId) throws ResourceUnavailableException {
          Commands cmds = new Commands(Command.OnError.Continue);
          createApplyLoadBalancingRulesCommands(rules, elbVm, cmds, guestNetworkId);
          // Send commands to elbVm
          return sendCommandsToRouter(elbVm, cmds);
      }
  
      protected DomainRouterVO findElbVmForLb(LoadBalancingRule lb) {//TODO: use a table to lookup
          Network ntwk = _networkModel.getNetwork(lb.getNetworkId());
          long sourceIpId = _networkModel.getPublicIpAddress(lb.getSourceIp().addr(), ntwk.getDataCenterId()).getId();
          ElasticLbVmMapVO map = _elbVmMapDao.findOneByIp(sourceIpId);
          if (map == null) {
              return null;
          }
          DomainRouterVO elbVm = _routerDao.findById(map.getElbVmId());
          return elbVm;
      }
  
      @Override
      public boolean applyLoadBalancerRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
          if (rules == null || rules.isEmpty()) {
              return true;
          }
  
          DomainRouterVO elbVm = findElbVmForLb(rules.get(0));
  
          if (elbVm == null) {
              logger.warn("Unable to apply lb rules, ELB vm doesn't exist in the network {}", network);
              throw new ResourceUnavailableException("Unable to apply lb rules", DataCenter.class, network.getDataCenterId());
          }
  
          if (elbVm.getState() == State.Running) {
              //resend all rules for the public ip
              long sourceIpId = _networkModel.getPublicIpAddress(rules.get(0).getSourceIp().addr(), network.getDataCenterId()).getId();
              List<LoadBalancerVO> lbs = _lbDao.listByIpAddress(sourceIpId);
              List<LoadBalancingRule> lbRules = new ArrayList<LoadBalancingRule>();
              for (LoadBalancerVO lb : lbs) {
                  List<LbDestination> dstList = _lbMgr.getExistingDestinations(lb.getId());
                  List<LbStickinessPolicy> policyList = _lbMgr.getStickinessPolicies(lb.getId());
                  List<LbHealthCheckPolicy> hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId());
                  Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress();
                  LbSslCert sslCert = _lbMgr.getLbSslCert(lb.getId());
                  LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp, sslCert, lb.getLbProtocol());
                  lbRules.add(loadBalancing);
              }
              return applyLBRules(elbVm, lbRules, network.getId());
          } else if (elbVm.getState() == State.Stopped || elbVm.getState() == State.Stopping) {
              logger.debug(String.format("ELB VM %s is in %s, so not sending apply LoadBalancing rules commands to the backend", elbVm, elbVm.getState()));
              return true;
          } else {
              logger.warn(String.format("Unable to apply loadbalancing rules, ELB VM %s is not in the right state %s", elbVm, elbVm.getState()));
              throw new ResourceUnavailableException("Unable to apply loadbalancing rules, ELB VM is not in the right state", VirtualRouter.class, elbVm.getId());
          }
      }
  
      @Override
      public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
          final Map<String, String> configs = _configDao.getConfiguration("AgentManager", params);
          _systemAcct = _accountService.getSystemAccount();
          _instance = configs.get("instance.name");
          if (_instance == null) {
              _instance = "VM";
          }
          _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key());
  
          _elasticLbVmRamSize = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmMemory.key()), DEFAULT_ELB_VM_RAMSIZE);
          _elasticLbvmCpuMHz = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmCpuMhz.key()), DEFAULT_ELB_VM_CPU_MHZ);
          _elasticLbvmNumCpu = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmNumVcpu.key()), 1);
          List<ServiceOfferingVO> offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Elastic LB VM",
                  ServiceOffering.elbVmDefaultOffUniqueName, _elasticLbvmNumCpu, _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null,
                  Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true);
          // this can sometimes happen, if DB is manually or programmatically manipulated
          if (offerings == null || offerings.size() < 2) {
              String msg = "Data integrity problem : System Offering For Elastic LB VM has been removed?";
              logger.error(msg);
              throw new ConfigurationException(msg);
          }
  
          String enabled = _configDao.getValue(Config.ElasticLoadBalancerEnabled.key());
          _enabled = (enabled == null) ? false : Boolean.parseBoolean(enabled);
          logger.info("Elastic Load balancer enabled: " + _enabled);
          if (_enabled) {
              String traffType = _configDao.getValue(Config.ElasticLoadBalancerNetwork.key());
              if ("guest".equalsIgnoreCase(traffType)) {
                  _frontendTrafficType = TrafficType.Guest;
              } else if ("public".equalsIgnoreCase(traffType)) {
                  _frontendTrafficType = TrafficType.Public;
              } else
                  throw new ConfigurationException("ELB: Traffic type for front end of load balancer has to be guest or public; found : " + traffType);
              logger.info("ELB: Elastic Load Balancer: will balance on " + traffType);
              int gcIntervalMinutes = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmGcInterval.key()), 5);
              if (gcIntervalMinutes < 5)
                  gcIntervalMinutes = 5;
              logger.info("ELB: Elastic Load Balancer: scheduling GC to run every " + gcIntervalMinutes + " minutes");
              _gcThreadPool = Executors.newScheduledThreadPool(1, new NamedThreadFactory("ELBVM-GC"));
              _gcThreadPool.scheduleAtFixedRate(new CleanupThread(), gcIntervalMinutes, gcIntervalMinutes, TimeUnit.MINUTES);
              _itMgr.registerGuru(VirtualMachine.Type.ElasticLoadBalancerVm, this);
          }
  
          loadBalanceRuleHandler = new LoadBalanceRuleHandler(_instance, _systemAcct);
  
          return true;
      }
  
      private DomainRouterVO stop(DomainRouterVO elbVm, boolean forced) throws ConcurrentOperationException, ResourceUnavailableException {
          logger.debug("Stopping ELB vm " + elbVm);
          try {
              _itMgr.advanceStop(elbVm.getUuid(), forced);
              return _routerDao.findById(elbVm.getId());
          } catch (OperationTimedoutException e) {
              throw new CloudRuntimeException("Unable to stop " + elbVm, e);
          }
      }
  
      @Override
      public LoadBalancer handleCreateLoadBalancerRule(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException,
              NetworkRuleConflictException {
          return loadBalanceRuleHandler.handleCreateLoadBalancerRule(lb, account, networkId);
      }
  
      void garbageCollectUnusedElbVms() {
          List<DomainRouterVO> unusedElbVms = _elbVmMapDao.listUnusedElbVms();
          if (unusedElbVms != null) {
              if (unusedElbVms.size() > 0) {
                  logger.info("Found " + unusedElbVms.size() + " unused ELB vms");
              }
              Set<Long> currentGcCandidates = new HashSet<Long>();
              for (DomainRouterVO elbVm : unusedElbVms) {
                  currentGcCandidates.add(elbVm.getId());
              }
              _gcCandidateElbVmIds.retainAll(currentGcCandidates);
              currentGcCandidates.removeAll(_gcCandidateElbVmIds);
              for (Long elbVmId : _gcCandidateElbVmIds) {
                  DomainRouterVO elbVm = _routerDao.findById(elbVmId);
                  boolean gceed = false;
  
                  try {
                      logger.info("Attempting to stop ELB VM: " + elbVm);
                      stop(elbVm, true);
                      gceed = true;
                  } catch (ConcurrentOperationException e) {
                      logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
                  } catch (ResourceUnavailableException e) {
                      logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
                      continue;
                  }
                  if (gceed) {
                      try {
                          logger.info("Attempting to destroy ELB VM: " + elbVm);
                          _itMgr.expunge(elbVm.getUuid());
                          _routerDao.remove(elbVm.getId());
                      } catch (ResourceUnavailableException e) {
                          logger.warn("Unable to destroy unused ELB vm " + elbVm + " due to ", e);
                          gceed = false;
                      }
                  }
                  if (!gceed) {
                      currentGcCandidates.add(elbVm.getId());
                  }
  
              }
              _gcCandidateElbVmIds = currentGcCandidates;
          }
      }
  
      public class CleanupThread extends ManagedContextRunnable {
          @Override
          protected void runInContext() {
              garbageCollectUnusedElbVms();
  
          }
  
          CleanupThread() {
  
          }
      }
  
      @Override
      public void handleDeleteLoadBalancerRule(LoadBalancer lb, long userId, Account caller) {
          if (!_enabled) {
              return;
          }
          loadBalanceRuleHandler.handleDeleteLoadBalancerRule(lb, userId, caller);
      }
  
      @Override
      public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) {
  
          List<NicProfile> elbNics = profile.getNics();
          Long guestNtwkId = null;
          for (NicProfile routerNic : elbNics) {
              if (routerNic.getTrafficType() == TrafficType.Guest) {
                  guestNtwkId = routerNic.getNetworkId();
                  break;
              }
          }
  
          NetworkVO guestNetwork = _networkDao.findById(guestNtwkId);
  
          DataCenter dc = dest.getDataCenter();
  
          StringBuilder buf = profile.getBootArgsBuilder();
          buf.append(" template=domP type=" + SystemVmType);
          buf.append(" name=").append(profile.getHostName());
          NicProfile controlNic = null;
          String defaultDns1 = null;
          String defaultDns2 = null;
  
          for (NicProfile nic : profile.getNics()) {
              int deviceId = nic.getDeviceId();
              buf.append(" eth").append(deviceId).append("ip=").append(nic.getIPv4Address());
              buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask());
              if (nic.isDefaultNic()) {
                  buf.append(" gateway=").append(nic.getIPv4Gateway());
                  defaultDns1 = nic.getIPv4Dns1();
                  defaultDns2 = nic.getIPv4Dns2();
              }
              if (nic.getTrafficType() == TrafficType.Management) {
                  buf.append(" localgw=").append(dest.getPod().getGateway());
              } else if (nic.getTrafficType() == TrafficType.Control) {
                  //  control command is sent over management network in VMware
                  if (dest.getHost().getHypervisorType() == HypervisorType.VMware) {
                      if (logger.isInfoEnabled()) {
                          logger.info("Check if we need to add management server explicit route to ELB vm. pod cidr: " + dest.getPod().getCidrAddress() + "/"
                                  + dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: "
                                  + ApiServiceConfiguration.ManagementServerAddresses.value());
                      }
  
                      if (logger.isDebugEnabled()) {
                          logger.debug("Added management server explicit route to ELB vm.");
                      }
                      // always add management explicit route, for basic networking setup
                      buf.append(" mgmtcidr=").append(_mgmtCidr);
                      buf.append(" localgw=").append(dest.getPod().getGateway());
  
                      if (dc.getNetworkType() == NetworkType.Basic) {
                          // ask elb vm to setup SSH on guest network
                          buf.append(" sshonguest=true");
                      }
  
                  }
  
                  controlNic = nic;
              }
          }
          String domain = guestNetwork.getNetworkDomain();
          if (domain != null) {
              buf.append(" domain=" + domain);
          }
  
          buf.append(" dns1=").append(defaultDns1);
          if (defaultDns2 != null) {
              buf.append(" dns2=").append(defaultDns2);
          }
          String msPublicKey = _configDao.getValue("ssh.publickey");
          buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey));
  
          if (SystemVmEnableUserData.valueIn(dc.getId())) {
              String userDataUuid = VirtualRouterUserData.valueIn(dc.getId());
              try {
                  String userData = userDataManager.validateAndGetUserDataForSystemVM(userDataUuid);
                  if (StringUtils.isNotBlank(userData)) {
                      buf.append(" userdata=").append(userData);
                  }
              } catch (Exception e) {
                  logger.warn("Failed to load user data for the elastic lb vm, ignored", e);
              }
          }
  
          if (logger.isDebugEnabled()) {
              logger.debug("Boot Args for " + profile + ": " + buf.toString());
          }
  
          if (controlNic == null) {
              throw new CloudRuntimeException("Didn't start a control port");
          }
  
          return true;
      }
  
      @Override
      public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException {
          DomainRouterVO elbVm = _routerDao.findById(profile.getVirtualMachine().getId());
  
          List<NicProfile> nics = profile.getNics();
          for (NicProfile nic : nics) {
              if (nic.getTrafficType() == TrafficType.Public) {
                  elbVm.setPublicIpAddress(nic.getIPv4Address());
                  elbVm.setPublicNetmask(nic.getIPv4Netmask());
                  elbVm.setPublicMacAddress(nic.getMacAddress());
              } else if (nic.getTrafficType() == TrafficType.Control) {
                  elbVm.setPrivateIpAddress(nic.getIPv4Address());
                  elbVm.setPrivateMacAddress(nic.getMacAddress());
              }
          }
          _routerDao.update(elbVm.getId(), elbVm);
  
          finalizeCommandsOnStart(cmds, profile);
          return true;
      }
  
      @Override
      public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) {
          CheckSshAnswer answer = (CheckSshAnswer)cmds.getAnswer("checkSsh");
          if (answer == null || !answer.getResult()) {
              logger.warn("Unable to ssh to the ELB VM: " + (answer != null ? answer.getDetails() : "No answer (answer for \"checkSsh\" was null)"));
              return false;
          }
  
          return true;
      }
  
      @Override
      public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) {
          DomainRouterVO elbVm = _routerDao.findById(profile.getVirtualMachine().getId());
          DataCenterVO dcVo = _dcDao.findById(elbVm.getDataCenterId());
  
          NicProfile controlNic = null;
          Long guestNetworkId = null;
  
          if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) {
              // TODO this is a ugly to test hypervisor type here
              // for basic network mode, we will use the guest NIC for control NIC
              for (NicProfile nic : profile.getNics()) {
                  if (nic.getTrafficType() == TrafficType.Guest && nic.getIPv4Address() != null) {
                      controlNic = nic;
                      guestNetworkId = nic.getNetworkId();
                  }
              }
          } else {
              for (NicProfile nic : profile.getNics()) {
                  if (nic.getTrafficType() == TrafficType.Control && nic.getIPv4Address() != null) {
                      controlNic = nic;
                  } else if (nic.getTrafficType() == TrafficType.Guest) {
                      guestNetworkId = nic.getNetworkId();
                  }
              }
          }
  
          if (controlNic == null) {
              logger.error("Control network doesn't exist for the ELB vm " + elbVm);
              return false;
          }
  
          cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIPv4Address(), 3922));
  
          // Re-apply load balancing rules
          List<LoadBalancerVO> lbs = _elbVmMapDao.listLbsForElbVm(elbVm.getId());
          List<LoadBalancingRule> lbRules = new ArrayList<LoadBalancingRule>();
          for (LoadBalancerVO lb : lbs) {
              List<LbDestination> dstList = _lbMgr.getExistingDestinations(lb.getId());
              List<LbStickinessPolicy> policyList = _lbMgr.getStickinessPolicies(lb.getId());
              List<LbHealthCheckPolicy> hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId());
              Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress();
              LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp);
              lbRules.add(loadBalancing);
          }
  
          logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of ELB vm " + elbVm + " start.");
          if (!lbRules.isEmpty()) {
              createApplyLoadBalancingRulesCommands(lbRules, elbVm, cmds, guestNetworkId);
          }
  
          return true;
      }
  
      @Override
      public void finalizeStop(VirtualMachineProfile profile, Answer answer) {
          if (answer != null) {
              DomainRouterVO elbVm = _routerDao.findById(profile.getVirtualMachine().getId());
              processStopOrRebootAnswer(elbVm, answer);
          }
      }
  
      @SuppressWarnings("unused")
      public void processStopOrRebootAnswer(final DomainRouterVO elbVm, Answer answer) {
          //TODO: process network usage stats
      }
  
      @Override
      public void finalizeExpunge(VirtualMachine vm) {
          // no-op
      }
  
      @Override
      public void prepareStop(VirtualMachineProfile profile) {
      }
  
      @Override
      public void finalizeUnmanage(VirtualMachine vm) {
      }
  
      @Override
      public void expungeLbVmRefs(List<Long> vmIds, Long batchSize) {
          _elbVmMapDao.expungeByLbVmList(vmIds, batchSize);
      }
  }
  