content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.alert;
  
  import java.io.UnsupportedEncodingException;
  import java.text.DecimalFormat;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Date;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.Timer;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ExecutionException;
  import java.util.concurrent.ExecutorService;
  import java.util.concurrent.Executors;
  import java.util.concurrent.Future;
  
  import javax.inject.Inject;
  import javax.mail.MessagingException;
  import javax.naming.ConfigurationException;
  
  import com.cloud.dc.DataCenter;
  import com.cloud.dc.Pod;
  import com.cloud.org.Cluster;
  
  import org.apache.cloudstack.backup.BackupManager;
  import org.apache.cloudstack.framework.config.ConfigDepot;
  import org.apache.cloudstack.framework.config.ConfigKey;
  import org.apache.cloudstack.framework.config.Configurable;
  import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
  import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
  import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
  import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
  import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
  import org.apache.cloudstack.utils.mailing.MailAddress;
  import org.apache.cloudstack.utils.mailing.SMTPMailProperties;
  import org.apache.cloudstack.utils.mailing.SMTPMailSender;
  import org.apache.commons.lang3.ArrayUtils;
  import org.apache.commons.lang3.math.NumberUtils;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  import org.jetbrains.annotations.Nullable;
  
  import com.cloud.alert.dao.AlertDao;
  import com.cloud.api.ApiDBUtils;
  import com.cloud.capacity.Capacity;
  import com.cloud.capacity.CapacityManager;
  import com.cloud.capacity.CapacityState;
  import com.cloud.capacity.CapacityVO;
  import com.cloud.capacity.dao.CapacityDao;
  import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity;
  import com.cloud.configuration.Config;
  import com.cloud.configuration.ConfigurationManager;
  import com.cloud.dc.ClusterVO;
  import com.cloud.dc.DataCenter.NetworkType;
  import com.cloud.dc.DataCenterVO;
  import com.cloud.dc.HostPodVO;
  import com.cloud.dc.Vlan.VlanType;
  import com.cloud.dc.dao.ClusterDao;
  import com.cloud.dc.dao.DataCenterDao;
  import com.cloud.dc.dao.DataCenterIpAddressDao;
  import com.cloud.dc.dao.HostPodDao;
  import com.cloud.event.ActionEvent;
  import com.cloud.event.AlertGenerator;
  import com.cloud.event.EventTypes;
  import com.cloud.host.Host;
  import com.cloud.host.HostVO;
  import com.cloud.host.dao.HostDao;
  import com.cloud.network.Ipv6Service;
  import com.cloud.network.dao.IPAddressDao;
  import com.cloud.org.Grouping.AllocationState;
  import com.cloud.resource.ResourceManager;
  import com.cloud.storage.StorageManager;
  import com.cloud.utils.Pair;
  import com.cloud.utils.component.ManagerBase;
  import com.cloud.utils.concurrency.NamedThreadFactory;
  import com.cloud.utils.db.SearchCriteria;
  import com.cloud.utils.db.Transaction;
  import com.cloud.utils.db.TransactionCallbackNoReturn;
  import com.cloud.utils.db.TransactionStatus;
  
  public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable {
      protected Logger logger = LogManager.getLogger(AlertManagerImpl.class.getName());
  
      public static final List<AlertType> ALERTS = Arrays.asList(AlertType.ALERT_TYPE_HOST
              , AlertType.ALERT_TYPE_USERVM
              , AlertType.ALERT_TYPE_DOMAIN_ROUTER
              , AlertType.ALERT_TYPE_CONSOLE_PROXY
              , AlertType.ALERT_TYPE_SSVM
              , AlertType.ALERT_TYPE_STORAGE_MISC
              , AlertType.ALERT_TYPE_MANAGEMENT_NODE
              , AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED
              , AlertType.ALERT_TYPE_UPLOAD_FAILED
              , AlertType.ALERT_TYPE_OOBM_AUTH_ERROR
              , AlertType.ALERT_TYPE_HA_ACTION
              , AlertType.ALERT_TYPE_CA_CERT
              , AlertType.ALERT_TYPE_EXTENSION_PATH_NOT_READY);
  
      private static final long INITIAL_CAPACITY_CHECK_DELAY = 30L * 1000L; // Thirty seconds expressed in milliseconds.
  
      private static final DecimalFormat DfPct = new DecimalFormat("###.##");
      private static final DecimalFormat DfWhole = new DecimalFormat("########");
  
      @Inject
      AlertDao _alertDao;
      @Inject
      protected StorageManager _storageMgr;
      @Inject
      protected CapacityManager _capacityMgr;
      @Inject
      CapacityDao _capacityDao;
      @Inject
      DataCenterDao _dcDao;
      @Inject
      HostPodDao _podDao;
      @Inject
      ClusterDao _clusterDao;
      @Inject
      IPAddressDao _publicIPAddressDao;
      @Inject
      DataCenterIpAddressDao _privateIPAddressDao;
      @Inject
      PrimaryDataStoreDao _storagePoolDao;
      @Inject
      ConfigurationDao _configDao;
      @Inject
      ResourceManager _resourceMgr;
      @Inject
      ConfigurationManager _configMgr;
      @Inject
      protected BackupManager backupManager;
      @Inject
      protected ConfigDepot _configDepot;
      @Inject
      private ObjectStoreDao _objectStoreDao;
      @Inject
      Ipv6Service ipv6Service;
      @Inject
      HostDao hostDao;
  
      private Timer _timer = null;
      private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default.
      private double _publicIPCapacityThreshold = 0.75;
      private double _privateIPCapacityThreshold = 0.75;
      private double _secondaryStorageCapacityThreshold = 0.75;
      private double _vlanCapacityThreshold = 0.75;
      private double _directNetworkPublicIpCapacityThreshold = 0.75;
      private double _localStorageCapacityThreshold = 0.75;
      private double _backupStorageCapacityThreshold = 0.75;
      private double _objectStorageCapacityThreshold = 0.75;
      Map<Short, Double> _capacityTypeThresholdMap = new HashMap<>();
  
      private final ExecutorService _executor;
  
      protected SMTPMailSender mailSender;
      protected String[] recipients = null;
      protected String senderAddress = null;
  
      public AlertManagerImpl() {
          _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Email-Alerts-Sender"));
      }
  
      @Override
      public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
          Map<String, String> configs = _configDao.getConfiguration("management-server", params);
  
          // set up the email system for alerts
          String emailAddressList = configs.get("alert.email.addresses");
          if (emailAddressList != null) {
              recipients = emailAddressList.split(",");
          }
  
          senderAddress = configs.get("alert.email.sender");
  
          String namespace = "alert.smtp";
          String timeoutConfig = String.format("%s.timeout", namespace);
          String connectionTimeoutConfig = String.format("%s.connectiontimeout", namespace);
  
          int smtpTimeout = NumberUtils.toInt(configs.get(timeoutConfig), 30000);
          int smtpConnectionTimeout = NumberUtils.toInt(configs.get(connectionTimeoutConfig), 30000);
  
          configs.put(timeoutConfig, String.valueOf(smtpTimeout));
          configs.put(connectionTimeoutConfig, String.valueOf(smtpConnectionTimeout));
  
          mailSender = new SMTPMailSender(configs, namespace);
  
          String publicIPCapacityThreshold = _configDao.getValue(Config.PublicIpCapacityThreshold.key());
          String privateIPCapacityThreshold = _configDao.getValue(Config.PrivateIpCapacityThreshold.key());
          String secondaryStorageCapacityThreshold = _configDao.getValue(Config.SecondaryStorageCapacityThreshold.key());
          String vlanCapacityThreshold = _configDao.getValue(Config.VlanCapacityThreshold.key());
          String directNetworkPublicIpCapacityThreshold = _configDao.getValue(Config.DirectNetworkPublicIpCapacityThreshold.key());
          String localStorageCapacityThreshold = _configDao.getValue(Config.LocalStorageCapacityThreshold.key());
          String backupStorageCapacityThreshold = _configDao.getValue(BackupManager.BackupStorageCapacityThreshold.key());
          String objectStorageCapacityThreshold = _configDao.getValue(_storageMgr.ObjectStorageCapacityThreshold.key());
  
          if (publicIPCapacityThreshold != null) {
              _publicIPCapacityThreshold = Double.parseDouble(publicIPCapacityThreshold);
          }
          if (privateIPCapacityThreshold != null) {
              _privateIPCapacityThreshold = Double.parseDouble(privateIPCapacityThreshold);
          }
          if (secondaryStorageCapacityThreshold != null) {
              _secondaryStorageCapacityThreshold = Double.parseDouble(secondaryStorageCapacityThreshold);
          }
          if (vlanCapacityThreshold != null) {
              _vlanCapacityThreshold = Double.parseDouble(vlanCapacityThreshold);
          }
          if (directNetworkPublicIpCapacityThreshold != null) {
              _directNetworkPublicIpCapacityThreshold = Double.parseDouble(directNetworkPublicIpCapacityThreshold);
          }
          if (localStorageCapacityThreshold != null) {
              _localStorageCapacityThreshold = Double.parseDouble(localStorageCapacityThreshold);
          }
          if (backupStorageCapacityThreshold != null) {
              _backupStorageCapacityThreshold = Double.parseDouble(backupStorageCapacityThreshold);
          }
          if (objectStorageCapacityThreshold != null) {
              _objectStorageCapacityThreshold = Double.parseDouble(objectStorageCapacityThreshold);
          }
  
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, _publicIPCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_PRIVATE_IP, _privateIPCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE, _secondaryStorageCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VLAN, _vlanCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, _directNetworkPublicIpCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_LOCAL_STORAGE, _localStorageCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, Ipv6SubnetCapacityThreshold.value());
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, _backupStorageCapacityThreshold);
          _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, _objectStorageCapacityThreshold);
  
          String capacityCheckPeriodStr = configs.get("capacity.check.period");
          if (capacityCheckPeriodStr != null) {
              _capacityCheckPeriod = Long.parseLong(capacityCheckPeriodStr);
              if (_capacityCheckPeriod <= 0) {
                  _capacityCheckPeriod = Long.parseLong(Config.CapacityCheckPeriod.getDefaultValue());
              }
          }
  
          _timer = new Timer("CapacityChecker");
  
          return true;
      }
  
      @Override
      public boolean start() {
          _timer.schedule(new CapacityChecker(), INITIAL_CAPACITY_CHECK_DELAY, _capacityCheckPeriod);
          return true;
      }
  
      @Override
      public boolean stop() {
          _timer.cancel();
          return true;
      }
  
      @Override
      public void clearAlert(AlertType alertType, long dataCenterId, long podId) {
          try {
              clearAlert(alertType.getType(), dataCenterId, podId);
          } catch (Exception ex) {
              logger.error("Problem clearing email alert", ex);
          }
      }
  
      @Override
      public void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body) {
  
          // publish alert
          AlertGenerator.publishAlertOnEventBus(alertType.getName(), dataCenterId, podId, subject, body);
  
          // TODO:  queue up these messages and send them as one set of issues once a certain number of issues is reached?  If that's the case,
          //         shouldn't we have a type/severity as part of the API so that severe errors get sent right away?
          try {
              if (mailSender != null) {
                  sendAlert(alertType, dataCenterId, podId, null, subject, body);
              } else {
                  logger.warn("AlertType:: " + alertType + " | dataCenterId:: " + dataCenterId + " | podId:: " + podId +
                          " | message:: " + subject + " | body:: " + body);
              }
          } catch (Exception ex) {
              logger.error("Problem sending email alert", ex);
          }
      }
  
      /**
       * Recalculates the capacities of hosts, including CPU and RAM.
       */
      protected void recalculateHostCapacities() {
          List<Long> hostIds = hostDao.listIdsByType(Host.Type.Routing);
          if (hostIds.isEmpty()) {
              return;
          }
          ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
          ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
                  Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size())));
          for (Long hostId : hostIds) {
              futures.put(hostId, executorService.submit(() -> {
                  Transaction.execute(new TransactionCallbackNoReturn() {
                      @Override
                      public void doInTransactionWithoutResult(TransactionStatus status) {
                          final HostVO host = hostDao.findById(hostId);
                          _capacityMgr.updateCapacityForHost(host);
                      }
                  });
                  return null;
              }));
          }
          for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
              try {
                  entry.getValue().get();
              } catch (InterruptedException | ExecutionException e) {
                  logger.error(String.format("Error during capacity calculation for host: %d due to : %s",
                          entry.getKey(), e.getMessage()), e);
              }
          }
          executorService.shutdown();
      }
  
      protected void recalculateStorageCapacities() {
          List<Long> storagePoolIds = _storagePoolDao.listAllIds();
          if (storagePoolIds.isEmpty()) {
              return;
          }
          ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
          ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
                  Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size())));
          for (Long poolId: storagePoolIds) {
              futures.put(poolId, executorService.submit(() -> {
                  Transaction.execute(new TransactionCallbackNoReturn() {
                      @Override
                      public void doInTransactionWithoutResult(TransactionStatus status) {
                          final StoragePoolVO pool = _storagePoolDao.findById(poolId);
                          long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null);
                          if (pool.isShared()) {
                              _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk);
                          } else {
                              _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk);
                          }
                      }
                  });
                  return null;
              }));
          }
          for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
              try {
                  entry.getValue().get();
              } catch (InterruptedException | ExecutionException e) {
                  logger.error(String.format("Error during capacity calculation for storage pool: %d due to : %s",
                          entry.getKey(), e.getMessage()), e);
              }
          }
          executorService.shutdown();
      }
  
      @Override
      public void recalculateCapacity() {
          // FIXME: the right way to do this is to register a listener (see RouterStatsListener, VMSyncListener)
          //        for the vm sync state.  The listener model has connects/disconnects to keep things in sync much better
          //        than this model right now, so when a VM is started, we update the amount allocated, and when a VM
          //        is stopped we updated the amount allocated, and when VM sync reports a changed state, we update
          //        the amount allocated.  Hopefully it's limited to 3 entry points and will keep the amount allocated
          //        per host accurate.
  
          try {
  
              if (logger.isDebugEnabled()) {
                  logger.debug("recalculating system capacity");
                  logger.debug("Executing cpu/ram capacity update");
              }
              // Calculate CPU and RAM capacities
              recalculateHostCapacities();
              if (logger.isDebugEnabled()) {
                  logger.debug("Done executing cpu/ram capacity update");
                  logger.debug("Executing storage capacity update");
              }
              // Calculate storage pool capacity
              recalculateStorageCapacities();
              if (logger.isDebugEnabled()) {
                  logger.debug("Done executing storage capacity update");
                  logger.debug("Executing capacity updates for public ip and Vlans");
              }
  
              List<DataCenterVO> datacenters = _dcDao.listAll();
              for (DataCenterVO datacenter : datacenters) {
                  long dcId = datacenter.getId();
  
                  //NOTE
                  //What happens if we have multiple vlans? Dashboard currently shows stats
                  //with no filter based on a vlan
                  //ideal way would be to remove out the vlan param, and filter only on dcId
                  //implementing the same
  
                  // Calculate new Public IP capacity for Virtual Network
                  if (datacenter.getNetworkType() == NetworkType.Advanced) {
                      createOrUpdateIpCapacity(dcId, null, Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, datacenter.getAllocationState());
                      createOrUpdateIpv6Capacity(dcId, datacenter.getAllocationState());
                  }
  
                  // Calculate new Public IP capacity for Direct Attached Network
                  createOrUpdateIpCapacity(dcId, null, Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, datacenter.getAllocationState());
  
                  if (datacenter.getNetworkType() == NetworkType.Advanced) {
                      //Calculate VLAN's capacity
                      createOrUpdateVlanCapacity(dcId, datacenter.getAllocationState());
                  }
              }
  
              if (logger.isDebugEnabled()) {
                  logger.debug("Done capacity updates for public ip and Vlans");
                  logger.debug("Executing capacity updates for private ip");
              }
  
              // Calculate new Private IP capacity
              List<HostPodVO> pods = _podDao.listAll();
              for (HostPodVO pod : pods) {
                  long podId = pod.getId();
                  long dcId = pod.getDataCenterId();
  
                  createOrUpdateIpCapacity(dcId, podId, Capacity.CAPACITY_TYPE_PRIVATE_IP, _configMgr.findPodAllocationState(pod));
              }
  
              if (logger.isDebugEnabled()) {
                  logger.debug("Done executing capacity updates for private ip");
                  logger.debug("Done recalculating system capacity");
              }
  
          } catch (Throwable t) {
              logger.error("Caught exception in recalculating capacity", t);
          }
      }
  
      private void createOrUpdateVlanCapacity(long dcId, AllocationState capacityState) {
  
          SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
          capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
          capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_VLAN);
          List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);
  
          int totalVlans = _dcDao.countZoneVlans(dcId, false);
          int allocatedVlans = _dcDao.countZoneVlans(dcId, true);
  
          CapacityState vlanCapacityState = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
          if (capacities.isEmpty()) {
              CapacityVO newVlanCapacity = new CapacityVO(null, dcId, null, null, allocatedVlans, totalVlans, Capacity.CAPACITY_TYPE_VLAN);
              newVlanCapacity.setCapacityState(vlanCapacityState);
              _capacityDao.persist(newVlanCapacity);
          } else if (!(capacities.get(0).getUsedCapacity() == allocatedVlans && capacities.get(0).getTotalCapacity() == totalVlans
                  && capacities.get(0).getCapacityState() == vlanCapacityState)) {
              CapacityVO capacity = capacities.get(0);
              capacity.setUsedCapacity(allocatedVlans);
              capacity.setTotalCapacity(totalVlans);
              capacity.setCapacityState(vlanCapacityState);
              _capacityDao.update(capacity.getId(), capacity);
          }
  
      }
  
      public void createOrUpdateIpCapacity(Long dcId, Long podId, short capacityType, AllocationState capacityState) {
          SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
          capacitySC.addAnd("podId", SearchCriteria.Op.EQ, podId);
          capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
          capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);
  
          int totalIPs;
          int allocatedIPs;
          List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);
          if (capacityType == Capacity.CAPACITY_TYPE_PRIVATE_IP) {
              totalIPs = _privateIPAddressDao.countIPs(podId, dcId, false);
              allocatedIPs = _privateIPAddressDao.countIPs(podId, dcId, true);
          } else if (capacityType == Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP) {
              totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.VirtualNetwork);
              allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.VirtualNetwork);
          } else {
              totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.DirectAttached);
              allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.DirectAttached);
          }
  
          CapacityState ipCapacityState = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
          if (capacities.isEmpty()) {
              CapacityVO newPublicIPCapacity = new CapacityVO(null, dcId, podId, null, allocatedIPs, totalIPs, capacityType);
              newPublicIPCapacity.setCapacityState(ipCapacityState);
              _capacityDao.persist(newPublicIPCapacity);
          } else if (!(capacities.get(0).getUsedCapacity() == allocatedIPs && capacities.get(0).getTotalCapacity() == totalIPs
                  && capacities.get(0).getCapacityState() == ipCapacityState)) {
              CapacityVO capacity = capacities.get(0);
              capacity.setUsedCapacity(allocatedIPs);
              capacity.setTotalCapacity(totalIPs);
              capacity.setCapacityState(ipCapacityState);
              _capacityDao.update(capacity.getId(), capacity);
          }
      }
  
      public void createOrUpdateIpv6Capacity(Long dcId, AllocationState capacityState) {
          final short capacityType = Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET;
          SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
          capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
          capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);
  
          List<CapacityVO>  capacities = _capacityDao.search(capacitySC, null);
          Pair<Integer, Integer> usedTotal =  ipv6Service.getUsedTotalIpv6SubnetForZone(dcId);
          int total = usedTotal.second();
          int allocated = usedTotal.first();
          CapacityState state = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
          if (capacities.isEmpty()) {
              CapacityVO capacityVO = new CapacityVO(null, dcId, null, null, allocated, total, capacityType);
              capacityVO.setCapacityState(state);
              _capacityDao.persist(capacityVO);
          } else if (!(capacities.get(0).getUsedCapacity() == allocated && capacities.get(0).getTotalCapacity() == total
                  && capacities.get(0).getCapacityState() == state)) {
              CapacityVO capacity = capacities.get(0);
              capacity.setUsedCapacity(allocated);
              capacity.setTotalCapacity(total);
              capacity.setCapacityState(state);
              _capacityDao.update(capacity.getId(), capacity);
          }
      }
  
      class CapacityChecker extends ManagedContextTimerTask {
          @Override
          protected void runInContext() {
              try {
                  logger.debug("Running Capacity Checker ... ");
                  checkForAlerts();
                  logger.debug("Done running Capacity Checker ... ");
              } catch (Throwable t) {
                  logger.error("Exception in CapacityChecker", t);
              }
          }
      }
  
      public void checkForAlerts() {
  
          recalculateCapacity();
  
          if (mailSender == null) {
              return;
          }
  
          //Get all datacenters, pods and clusters in the system.
          List<DataCenterVO> dataCenterList = _dcDao.listAll();
          List<ClusterVO> clusterList = _clusterDao.listAll();
          List<HostPodVO> podList = _podDao.listAll();
          //Get capacity types at different levels
          List<Short> dataCenterCapacityTypes = getCapacityTypesAtZoneLevel();
          List<Short> podCapacityTypes = getCapacityTypesAtPodLevel();
          List<Short> clusterCapacityTypes = getCapacityTypesAtClusterLevel();
  
          // Generate Alerts for Zone Level capacities
          for (DataCenterVO dc : dataCenterList) {
              for (Short capacityType : dataCenterCapacityTypes) {
                  List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), dc.getId(), null, null);
  
                  if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE ||
                      capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE ||
                      capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) {
                      capacity.add(getUsedStats(capacityType, dc.getId(), null, null));
                  }
                  if (capacity == null || capacity.isEmpty()) {
                      continue;
                  }
                  double totalCapacity = capacity.get(0).getTotalCapacity();
                  double usedCapacity = capacity.get(0).getUsedCapacity();
                  if (totalCapacity != 0 && usedCapacity / totalCapacity > _capacityTypeThresholdMap.get(capacityType)) {
                      generateEmailAlert(dc, null, null, totalCapacity, usedCapacity, capacityType);
                  }
              }
          }
  
          // Generate Alerts for Pod Level capacities
          for (HostPodVO pod : podList) {
              for (Short capacityType : podCapacityTypes) {
                  List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), pod.getDataCenterId(), pod.getId(), null);
                  if (capacity == null || capacity.isEmpty()) {
                      continue;
                  }
                  double totalCapacity = capacity.get(0).getTotalCapacity();
                  double usedCapacity = capacity.get(0).getUsedCapacity();
                  if (totalCapacity != 0 && usedCapacity / totalCapacity > _capacityTypeThresholdMap.get(capacityType)) {
                      generateEmailAlert(ApiDBUtils.findZoneById(pod.getDataCenterId()), pod, null, totalCapacity, usedCapacity, capacityType);
                  }
              }
          }
  
          // Generate Alerts for Cluster Level capacities
          for (ClusterVO cluster : clusterList) {
              for (Short capacityType : clusterCapacityTypes) {
                  List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), cluster.getDataCenterId(), null, cluster.getId());
  
                  // cpu and memory allocated capacity notification threshold can be defined at cluster level, so getting the value if they are defined at cluster level
                  double threshold;
                  switch (capacityType) {
                  case Capacity.CAPACITY_TYPE_STORAGE:
                      capacity.add(getUsedStats(capacityType, cluster.getDataCenterId(), cluster.getPodId(), cluster.getId()));
                      threshold = StorageCapacityThreshold.valueIn(cluster.getId());
                      break;
                  case Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED:
                      threshold = StorageAllocatedCapacityThreshold.valueIn(cluster.getId());
                      break;
                  case Capacity.CAPACITY_TYPE_CPU:
                      threshold = CPUCapacityThreshold.valueIn(cluster.getId());
                      break;
                  case Capacity.CAPACITY_TYPE_MEMORY:
                      threshold = MemoryCapacityThreshold.valueIn(cluster.getId());
                      break;
                  default:
                      threshold = _capacityTypeThresholdMap.get(capacityType);
                  }
                  if (capacity == null || capacity.isEmpty()) {
                      continue;
                  }
  
                  double totalCapacity = capacity.get(0).getTotalCapacity();
                  double usedCapacity = capacity.get(0).getUsedCapacity() + capacity.get(0).getReservedCapacity();
                  if (totalCapacity != 0 && usedCapacity / totalCapacity > threshold) {
                      generateEmailAlert(ApiDBUtils.findZoneById(cluster.getDataCenterId()), ApiDBUtils.findPodById(cluster.getPodId()), cluster, totalCapacity,
                              usedCapacity, capacityType);
                  }
              }
          }
  
      }
  
      private SummedCapacity getUsedStats(short capacityType, long zoneId, Long podId, Long clusterId) {
          CapacityVO capacity = null;
  
          if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) {
              capacity = _storageMgr.getSecondaryStorageUsedStats(null, zoneId);
          } else if (capacityType == Capacity.CAPACITY_TYPE_STORAGE) {
              capacity = _storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId);
          } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) {
              capacity = _storageMgr.getObjectStorageUsedStats(zoneId);
          } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) {
              capacity = (CapacityVO) backupManager.getBackupStorageUsedStats(zoneId);
          }
          if (capacity != null) {
              return new SummedCapacity(capacity.getUsedCapacity(), 0, capacity.getTotalCapacity(), capacityType, clusterId, podId);
          } else {
              return null;
          }
      }
  
      private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluster, double totalCapacity, double usedCapacity, short capacityType) {
  
          String msgSubject = null;
          String msgContent = null;
          String percentual = formatPercent(usedCapacity / totalCapacity);
          String totalInMB = formatBytesToMegabytes(totalCapacity);
          String usedInMB = formatBytesToMegabytes(usedCapacity);
          String totalInString = String.valueOf(totalCapacity);
          String usedInString = String.valueOf(usedCapacity);
          AlertType alertType = null;
          Long podId = pod == null ? null : pod.getId();
          Long clusterId = cluster == null ? null : cluster.getId();
          String clusterName = cluster == null ? null : cluster.getName();
          String podName = pod == null ? null : pod.getName();
          String dataCenterName = dc.getName();
  
          switch (capacityType) {
              case Capacity.CAPACITY_TYPE_MEMORY:
                  msgSubject = String.format("System Alert: Low Available Memory in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                  msgContent = String.format("System memory is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_MEMORY;
                  break;
              case Capacity.CAPACITY_TYPE_CPU:
                  msgSubject = String.format("System Alert: Low Unallocated CPU in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                  msgContent = String.format("Unallocated CPU is low, total: %s Mhz, used: %s Mhz (%s%%).", DfWhole.format(totalCapacity), DfWhole.format(usedCapacity), percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_CPU;
                  break;
              case Capacity.CAPACITY_TYPE_STORAGE:
                  msgSubject = String.format("System Alert: Low Available Storage in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                  msgContent = String.format("Available storage space is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_STORAGE;
                  break;
              case Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED:
                  msgSubject = String.format("System Alert: Remaining unallocated Storage is low in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName,
                          dataCenterName);
                  msgContent = String.format("Unallocated storage space is low, total: %s MB, allocated: %s MB (%s%%)", totalInMB, usedInMB, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_STORAGE_ALLOCATED;
                  break;
              case Capacity.CAPACITY_TYPE_LOCAL_STORAGE:
                  msgSubject = String.format("System Alert: Remaining unallocated Local Storage is low in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName,
                          dataCenterName);
                  msgContent = String.format("Unallocated storage space is low, total: %s MB, allocated: %s MB (%s%%)", totalInMB, usedInMB, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_LOCAL_STORAGE;
                  break;
              case Capacity.CAPACITY_TYPE_PRIVATE_IP:
                  msgSubject = String.format("System Alert: Number of unallocated private IPs is low in pod %s of availability zone [%s].", podName, dataCenterName);
                  msgContent = String.format("Number of unallocated private IPs is low, total: %s, allocated: %s (%s%%)", totalInString, usedInString, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_PRIVATE_IP;
                  break;
              case Capacity.CAPACITY_TYPE_SECONDARY_STORAGE:
                  msgSubject = String.format("System Alert: Low Available Secondary Storage in availability zone [%s].", dataCenterName);
                  msgContent = String.format("Available secondary storage space is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_SECONDARY_STORAGE;
                  break;
              case Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP:
                  msgSubject = String.format("System Alert: Number of unallocated virtual network public IPs is low in availability zone [%s].", dataCenterName);
                  msgContent = String.format("Number of unallocated public IPs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_PUBLIC_IP;
                  break;
              case Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP:
                  msgSubject = String.format("System Alert: Number of unallocated shared network IPs is low in availability zone [%s].", dataCenterName);
                  msgContent = String.format("Number of unallocated shared network IPs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP;
                  break;
              case Capacity.CAPACITY_TYPE_VLAN:
                  msgSubject = String.format("System Alert: Number of unallocated VLANs is low in availability zone [%s].", dataCenterName);
                  msgContent = String.format("Number of unallocated VLANs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_VLAN;
                  break;
              case Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET:
                  msgSubject = String.format("System Alert: Number of unallocated virtual network guest IPv6 subnets is low in availability zone [%s].", dc.getName());
                  msgContent = String.format("Number of unallocated virtual network guest IPv6 subnets is low, total: [%s], allocated: [%s] (%s%%).", totalInString, usedInString, percentual);
                  alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET;
                  break;
              case Capacity.CAPACITY_TYPE_BACKUP_STORAGE:
                  msgSubject = "System Alert: Low Available Backup Storage in availability zone " + dc.getName();
                  msgContent = "Available backup storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)";
                  alertType = AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE;
                  break;
              case Capacity.CAPACITY_TYPE_OBJECT_STORAGE:
                  msgSubject = "System Alert: Low Available Object Storage in availability zone " + dc.getName();
                  msgContent = "Available object storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)";
                  alertType = AlertManager.AlertType.ALERT_TYPE_OBJECT_STORAGE;
                  break;
          }
  
          try {
              logger.debug("Sending alert with subject [{}] and content [{}].", msgSubject, msgContent);
              sendAlert(alertType, dc.getId(), podId, clusterId, msgSubject, msgContent);
          } catch (Exception ex) {
              logger.error("Exception in CapacityChecker", ex);
          }
      }
  
      private List<Short> getCapacityTypesAtZoneLevel() {
  
          List<Short> dataCenterCapacityTypes = new ArrayList<>();
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VLAN);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_BACKUP_STORAGE);
          dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_OBJECT_STORAGE);
          return dataCenterCapacityTypes;
  
      }
  
      private List<Short> getCapacityTypesAtPodLevel() {
  
          List<Short> podCapacityTypes = new ArrayList<>();
          podCapacityTypes.add(Capacity.CAPACITY_TYPE_PRIVATE_IP);
          return podCapacityTypes;
  
      }
  
      private List<Short> getCapacityTypesAtClusterLevel() {
  
          List<Short> clusterCapacityTypes = new ArrayList<>();
          clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_CPU);
          clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_MEMORY);
          clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE);
          clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED);
          clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_LOCAL_STORAGE);
          return clusterCapacityTypes;
  
      }
  
      public void clearAlert(short alertType, long dataCenterId, Long podId) {
          if (alertType != -1) {
              AlertVO alert = _alertDao.getLastAlert(alertType, dataCenterId, podId, null);
              if (alert != null) {
                  AlertVO updatedAlert = _alertDao.createForUpdate();
                  updatedAlert.setResolved(new Date());
                  _alertDao.update(alert.getId(), updatedAlert);
              }
          }
      }
  
      public void sendAlert(AlertType alertType, long dataCenterId, Long podId, Long clusterId, String subject, String content)
              throws MessagingException, UnsupportedEncodingException {
          DataCenterVO zone = _dcDao.findById(dataCenterId);
          HostPodVO pod = podId == null ? null : _podDao.findById(podId);
          ClusterVO cluster = clusterId == null ? null : _clusterDao.findById(clusterId);
          sendAlert(alertType, zone, pod, cluster, subject, content);
      }
  
      public void sendAlert(AlertType alertType, DataCenter dataCenter, Pod pod, Cluster cluster, String subject, String content)
              throws MessagingException, UnsupportedEncodingException {
          Long clusterId = cluster == null ? null : cluster.getId();
          Long podId = pod == null ? null : pod.getId();
          long dcId = dataCenter == null ? 0L : dataCenter.getId();
          logger.warn(String.format("alertType=[%s] dataCenterId=[%s] podId=[%s] clusterId=[%s] message=[%s].", alertType, dcId, podId, clusterId, subject));
          AlertVO alert = getAlertForTrivialAlertType(alertType, dcId, podId, clusterId);
  
          if (alert == null) {
              AlertVO newAlert = new AlertVO();
              newAlert.setType(alertType.getType());
              newAlert.setSubject(subject);
              newAlert.setContent(content);
              newAlert.setClusterId(clusterId);
              newAlert.setPodId(podId);
              newAlert.setDataCenterId(dcId);
              newAlert.setSentCount(1);
              newAlert.setLastSent(new Date());
              newAlert.setName(alertType.getName());
              _alertDao.persist(newAlert);
          } else {
              logger.debug("Have already sent: " + alert.getSentCount() + " emails for alert type '" + alertType + "' -- skipping send email");
              return;
          }
  
          if (ArrayUtils.isEmpty(recipients)) {
              logger.warn(String.format("No recipients set in global setting 'alert.email.addresses', "
                      + "skipping sending alert with subject [%s] and content [%s].", subject, content));
              return;
          }
  
          SMTPMailProperties mailProps = new SMTPMailProperties();
          mailProps.setSender(new MailAddress(senderAddress));
          mailProps.setSubject(subject);
          mailProps.setContent(content);
          mailProps.setContentType("text/plain");
  
          Set<MailAddress> addresses = new HashSet<>();
          for (String recipient : recipients) {
              addresses.add(new MailAddress(recipient));
          }
  
          mailProps.setRecipients(addresses);
  
          sendMessage(mailProps);
  
      }
  
      @Nullable
      private AlertVO getAlertForTrivialAlertType(AlertType alertType, long dataCenterId, Long podId, Long clusterId) {
          AlertVO alert = null;
          if (!ALERTS.contains(alertType)) {
              alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId);
          }
          return alert;
      }
  
      protected void sendMessage(SMTPMailProperties mailProps) {
          _executor.execute(new Runnable() {
              @Override
              public void run() {
                  mailSender.sendMail(mailProps);
              }
          });
      }
  
      private static String formatPercent(double percentage) {
          return DfPct.format(percentage * 100);
      }
  
      private static String formatBytesToMegabytes(double bytes) {
          double megaBytes = (bytes / (1024 * 1024));
          return DfWhole.format(megaBytes);
      }
  
      @Override
      public String getConfigComponentName() {
          return AlertManager.class.getSimpleName();
      }
  
      @Override
      public ConfigKey<?>[] getConfigKeys() {
          return new ConfigKey<?>[] {CPUCapacityThreshold, MemoryCapacityThreshold, StorageAllocatedCapacityThreshold, StorageCapacityThreshold, AlertSmtpEnabledSecurityProtocols,
              AlertSmtpUseStartTLS, Ipv6SubnetCapacityThreshold, AlertSmtpUseAuth};
      }
  
      @Override
      @ActionEvent(eventType = EventTypes.ALERT_GENERATE, eventDescription = "generating alert", async = true)
      public boolean generateAlert(AlertType alertType, long dataCenterId, Long podId, String msg) {
          try {
              sendAlert(alertType, dataCenterId, podId, msg, msg);
              return true;
          } catch (Exception ex) {
              logger.warn("Failed to generate an alert of type=" + alertType + "; msg=" + msg);
              return false;
          }
      }
  }
  