content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package org.apache.cloudstack.sioc;
  
  import java.util.ArrayList;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  
  import javax.inject.Inject;
  
  import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
  import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
  import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
  import org.apache.cloudstack.util.LoginInfo;
  import org.apache.cloudstack.util.vmware.VMwareUtil;
  import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
  import org.apache.logging.log4j.Logger;
  import org.apache.logging.log4j.LogManager;
  import org.springframework.stereotype.Component;
  
  import com.cloud.dc.DataCenterVO;
  import com.cloud.dc.dao.DataCenterDao;
  import com.cloud.dc.VmwareDatacenterVO;
  import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO;
  import com.cloud.dc.dao.VmwareDatacenterDao;
  import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao;
  import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder;
  import com.cloud.storage.Storage.StoragePoolType;
  import com.cloud.storage.Volume;
  import com.cloud.storage.VolumeVO;
  import com.cloud.storage.dao.DiskOfferingDao;
  import com.cloud.storage.dao.VolumeDao;
  import com.cloud.utils.db.GlobalLock;
  import com.cloud.vm.VMInstanceVO;
  import com.cloud.vm.dao.VMInstanceDao;
  
  import com.vmware.vim25.ManagedObjectReference;
  import com.vmware.vim25.SharesInfo;
  import com.vmware.vim25.SharesLevel;
  import com.vmware.vim25.StorageIOAllocationInfo;
  import com.vmware.vim25.VirtualDevice;
  import com.vmware.vim25.VirtualDeviceConfigSpec;
  import com.vmware.vim25.VirtualDeviceConfigSpecOperation;
  import com.vmware.vim25.VirtualDisk;
  import com.vmware.vim25.VirtualDeviceFileBackingInfo;
  import com.vmware.vim25.VirtualMachineConfigInfo;
  import com.vmware.vim25.VirtualMachineConfigSpec;
  
  @Component
  public class SiocManagerImpl implements SiocManager {
      protected Logger logger = LogManager.getLogger(getClass());
      private static final int LOCK_TIME_IN_SECONDS = 3;
      private static final int ONE_GB_IN_BYTES = 1000000000;
      private static final int LOWEST_SHARES_PER_VIRTUAL_DISK = 2000; // We want this to be greater than 1,000, which is the VMware default value.
      private static final int HIGHEST_SHARES_PER_VIRTUAL_DISK = 4000; // VMware limit
      private static final int LOWEST_LIMIT_IOPS_PER_VIRTUAL_DISK = 16; // VMware limit
      private static final int HIGHEST_LIMIT_IOPS_PER_VIRTUAL_DISK = 2147483647; // VMware limit
  
      @Inject private DataCenterDao zoneDao;
      @Inject private DiskOfferingDao diskOfferingDao;
      @Inject private PrimaryDataStoreDao storagePoolDao;
      @Inject private StoragePoolDetailsDao storagePoolDetailsDao;
      @Inject private VMInstanceDao vmInstanceDao;
      @Inject private VmwareDatacenterDao vmwareDcDao;
      @Inject private VmwareDatacenterZoneMapDao vmwareDcZoneMapDao;
      @Inject private VolumeDao volumeDao;
  
      @Override
      public void updateSiocInfo(long zoneId, long storagePoolId, int sharesPerGB, int limitIopsPerGB, int iopsNotifyThreshold) throws Exception {
          logger.info("'SiocManagerImpl.updateSiocInfo(long, long, int, int, int)' method invoked");
  
          DataCenterVO zone = zoneDao.findById(zoneId);
  
          if (zone == null) {
              throw new Exception("Error: No zone could be located for the following zone ID: " + zoneId + ".");
          }
  
          StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
  
          if (storagePool == null) {
              throw new Exception("Error: No storage pool could be located for the following pool ID: " + storagePoolId + ".");
          }
  
          if (storagePool.getDataCenterId() != zoneId) {
              throw new Exception(String.format("Error: Storage pool %s is not in zone %s.", storagePool, zone));
          }
  
          if (!storagePool.getPoolType().equals(StoragePoolType.VMFS)) {
              throw new Exception(String.format("Error: Storage pool %s does not represent a VMFS datastore.", storagePool));
          }
  
          String lockName = zone.getUuid() + "-" + storagePool.getUuid();
          GlobalLock lock = GlobalLock.getInternLock(lockName);
  
          if (!lock.lock(LOCK_TIME_IN_SECONDS)) {
              throw new Exception("Busy: The system is already processing this request.");
          }
  
          VMwareUtil.VMwareConnection connection = null;
  
          try {
              connection = VMwareUtil.getVMwareConnection(getLoginInfo(zoneId));
  
              Map<String, ManagedObjectReference> nameToVm = VMwareUtil.getVms(connection);
  
              List<ManagedObjectReference> allTasks = new ArrayList<>();
  
              int limitIopsTotal = 0;
  
              List<VolumeVO> volumes = volumeDao.findByPoolId(storagePoolId, null);
  
              if (volumes != null && volumes.size() > 0) {
                  Set<Long> instanceIds = new HashSet<>();
  
                  for (VolumeVO volume : volumes) {
                      Long instanceId = volume.getInstanceId();
  
                      if (instanceId != null) {
                          instanceIds.add(instanceId);
                      }
                  }
  
                  for (Long instanceId : instanceIds) {
                      ResultWrapper resultWrapper = updateSiocInfo(connection, nameToVm, instanceId, storagePool, sharesPerGB, limitIopsPerGB);
  
                      limitIopsTotal += resultWrapper.getLimitIopsTotal();
  
                      allTasks.addAll(resultWrapper.getTasks());
                  }
              }
              /*
              Set<String> vmNames = nameToVm.keySet();
  
              for (String vmName : vmNames) {
                  // If the VM's name doesn't start with "i-", then it should be a worker VM (which is not stored in the CloudStack datastore).
                  if (!vmName.startsWith("i-")) {
                      ResultWrapper resultWrapper = updateSiocInfoForWorkerVM(connection, nameToVm.get(vmName),
                              getDatastoreName(storagePool.getPath()), limitIopsPerGB);
  
                      limitIopsTotal += resultWrapper.getLimitIopsTotal();
  
                      allTasks.addAll(resultWrapper.getTasks());
                  }
              }
              */
              for (ManagedObjectReference task : allTasks) {
                  VMwareUtil.waitForTask(connection, task);
              }
  
              if (limitIopsTotal > iopsNotifyThreshold) {
                  throw new Exception("Warning: Total number of IOPS: " + limitIopsTotal + "; IOPS notify threshold: " + iopsNotifyThreshold);
              }
          }
          finally {
              VMwareUtil.closeVMwareConnection(connection);
  
              lock.unlock();
              lock.releaseRef();
          }
      }
  
      private ResultWrapper updateSiocInfo(VMwareUtil.VMwareConnection connection, Map<String, ManagedObjectReference> nameToVm, Long instanceId,
                                           StoragePoolVO storagePool, int sharesPerGB, int limitIopsPerGB) throws Exception {
          int limitIopsTotal = 0;
          List<ManagedObjectReference> tasks = new ArrayList<>();
  
          VMInstanceVO vmInstance = vmInstanceDao.findById(instanceId);
  
          if (vmInstance == null) {
              String errMsg = "Error: The VM with ID " + instanceId + " could not be located.";
  
              throw new Exception(errMsg);
          }
  
          String vmName = vmInstance.getInstanceName();
  
          ManagedObjectReference morVm = nameToVm.get(vmName);
  
          if (morVm == null) {
              String errMsg = String.format("Error: The VM %s could not be located (ManagedObjectReference).", vmInstance);
  
              throw new Exception(errMsg);
          }
  
          VirtualMachineConfigInfo vmci = (VirtualMachineConfigInfo)VMwareUtil.getEntityProps(connection, morVm,
                  new String[] { "config" }).get("config");
          List<VirtualDevice> devices = vmci.getHardware().getDevice();
  
          for (VirtualDevice device : devices) {
              if (device instanceof VirtualDisk) {
                  VirtualDisk disk = (VirtualDisk)device;
  
                  VolumeVO volumeVO = getVolumeFromVirtualDisk(vmInstance, storagePool.getId(), devices, disk);
  
                  if (volumeVO != null) {
                      boolean diskUpdated = false;
  
                      StorageIOAllocationInfo sioai = disk.getStorageIOAllocation();
  
                      SharesInfo sharesInfo = sioai.getShares();
  
                      int currentShares = sharesInfo.getShares();
                      int newShares = getNewSharesBasedOnVolumeSize(volumeVO, sharesPerGB);
  
                      if (currentShares != newShares) {
                          sharesInfo.setLevel(SharesLevel.CUSTOM);
                          sharesInfo.setShares(newShares);
  
                          diskUpdated = true;
                      }
  
                      long currentLimitIops = sioai.getLimit() !=  null ? sioai.getLimit() : Long.MIN_VALUE;
                      long newLimitIops = getNewLimitIopsBasedOnVolumeSize(volumeVO, limitIopsPerGB);
  
                      limitIopsTotal += newLimitIops;
  
                      if (currentLimitIops != newLimitIops) {
                          sioai.setLimit(newLimitIops);
  
                          diskUpdated = true;
                      }
  
                      if (diskUpdated) {
                          VirtualDeviceConfigSpec vdcs = new VirtualDeviceConfigSpec();
  
                          vdcs.setDevice(disk);
                          vdcs.setOperation(VirtualDeviceConfigSpecOperation.EDIT);
  
                          VirtualMachineConfigSpec vmcs = new VirtualMachineConfigSpec();
  
                          vmcs.getDeviceChange().add(vdcs);
  
                          try {
                              ManagedObjectReference task = VMwareUtil.reconfigureVm(connection, morVm, vmcs);
  
                              tasks.add(task);
  
                              logger.info(getInfoMsg(volumeVO, newShares, newLimitIops));
                          } catch (Exception ex) {
                              throw new Exception("Error: " + ex.getMessage());
                          }
                      }
                  }
              }
          }
  
          return new ResultWrapper(limitIopsTotal, tasks);
      }
  
      private String getDatastoreName(String path) throws Exception {
          String searchString = "/";
  
          int lastIndexOf = path.lastIndexOf(searchString);
  
          if (lastIndexOf == -1) {
              throw new Exception("Error: Invalid datastore path");
          }
  
          return path.substring(lastIndexOf + searchString.length());
      }
  
      private ResultWrapper updateSiocInfoForWorkerVM(VMwareUtil.VMwareConnection connection, ManagedObjectReference morVm, String datastoreName,
                                                      int limitIopsPerGB) throws Exception {
          int limitIopsTotal = 0;
          List<ManagedObjectReference> tasks = new ArrayList<>();
  
          VirtualMachineConfigInfo vmci = (VirtualMachineConfigInfo)VMwareUtil.getEntityProps(connection, morVm,
                  new String[] { "config" }).get("config");
          List<VirtualDevice> devices = vmci.getHardware().getDevice();
  
          for (VirtualDevice device : devices) {
              if (device instanceof VirtualDisk) {
                  VirtualDisk disk = (VirtualDisk)device;
  
                  if (disk.getBacking() instanceof VirtualDeviceFileBackingInfo) {
                      VirtualDeviceFileBackingInfo backingInfo = (VirtualDeviceFileBackingInfo)disk.getBacking();
  
                      if (backingInfo.getFileName().contains(datastoreName)) {
                          boolean diskUpdated = false;
  
                          StorageIOAllocationInfo sioai = disk.getStorageIOAllocation();
  
                          long currentLimitIops = sioai.getLimit() !=  null ? sioai.getLimit() : Long.MIN_VALUE;
                          long newLimitIops = getNewLimitIopsBasedOnVolumeSize(disk.getCapacityInBytes(), limitIopsPerGB);
  
                          limitIopsTotal += newLimitIops;
  
                          if (currentLimitIops != newLimitIops) {
                              sioai.setLimit(newLimitIops);
  
                              diskUpdated = true;
                          }
  
                          if (diskUpdated) {
                              VirtualDeviceConfigSpec vdcs = new VirtualDeviceConfigSpec();
  
                              vdcs.setDevice(disk);
                              vdcs.setOperation(VirtualDeviceConfigSpecOperation.EDIT);
  
                              VirtualMachineConfigSpec vmcs = new VirtualMachineConfigSpec();
  
                              vmcs.getDeviceChange().add(vdcs);
  
                              try {
                                  ManagedObjectReference task = VMwareUtil.reconfigureVm(connection, morVm, vmcs);
  
                                  tasks.add(task);
  
                                  logger.info(getInfoMsgForWorkerVm(newLimitIops));
                              } catch (Exception ex) {
                                  throw new Exception("Error: " + ex.getMessage());
                              }
                          }
                      }
                  }
              }
          }
  
          return new ResultWrapper(limitIopsTotal, tasks);
      }
  
      private String getInfoMsg(Volume volume, Integer newShares, Long newLimitIops) {
          String msgPrefix = String.format("VMware SIOC: Volume %s", volume);
  
          String msgNewShares = newShares != null ? "; New Shares = " + newShares : "";
  
          String msgNewLimitIops = newLimitIops != null ? "; New Limit IOPS = " + newLimitIops : "";
  
          return msgPrefix + msgNewShares + msgNewLimitIops;
      }
  
      private String getInfoMsgForWorkerVm(Long newLimitIops) {
          return "VMware SIOC: Worker VM's Limit IOPS set to " + newLimitIops;
      }
  
      private VolumeVO getVolumeFromVirtualDisk(VMInstanceVO vmInstance, long storagePoolId, List<VirtualDevice> allDevices,
              VirtualDisk disk) throws Exception {
          List<VolumeVO> volumes = volumeDao.findByInstance(vmInstance.getId());
  
          if (volumes == null || volumes.size() == 0) {
              String errMsg = String.format("Error: The VMware virtual disk '%s' could not be mapped to a CloudStack volume. There were no volumes for the VM: %s.", disk, vmInstance);
  
              throw new Exception(errMsg);
          }
  
          VirtualMachineDiskInfoBuilder diskInfoBuilder = VMwareUtil.getDiskInfoBuilder(allDevices);
  
          for (VolumeVO volume : volumes) {
              Long poolId = volume.getPoolId();
  
              if (poolId != null && poolId == storagePoolId) {
                  StoragePoolVO storagePool = storagePoolDao.findById(poolId);
                  String path = storagePool.getPath();
                  String charToSearchFor = "/";
                  int index = path.lastIndexOf(charToSearchFor) + charToSearchFor.length();
                  String datastoreName = path.substring(index);
                  VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(volume.getPath(), datastoreName);
  
                  if (diskInfo != null) {
                      String deviceBusName = VMwareUtil.getDeviceBusName(allDevices, disk);
  
                      if (deviceBusName.equals(diskInfo.getDiskDeviceBusName())) {
                          return volume;
                      }
                  }
              }
          }
  
          return null;
      }
  
      private int getNewSharesBasedOnVolumeSize(VolumeVO volumeVO, int sharesPerGB) {
          long volumeSizeInBytes = getVolumeSizeInBytes(volumeVO);
  
          double sizeInGB = volumeSizeInBytes / (double)ONE_GB_IN_BYTES;
  
          int shares = LOWEST_SHARES_PER_VIRTUAL_DISK + ((int)(sharesPerGB * sizeInGB));
  
          return getAdjustedShares(shares);
      }
  
      private int getAdjustedShares(int shares) {
          shares = Math.max(shares, LOWEST_SHARES_PER_VIRTUAL_DISK);
          shares = Math.min(shares, HIGHEST_SHARES_PER_VIRTUAL_DISK);
  
          return shares;
      }
  
      private long getNewLimitIopsBasedOnVolumeSize(VolumeVO volumeVO, int limitIopsPerGB) {
          long volumeSizeInBytes = getVolumeSizeInBytes(volumeVO);
  
          return getNewLimitIopsBasedOnVolumeSize(volumeSizeInBytes, limitIopsPerGB);
      }
  
      private long getNewLimitIopsBasedOnVolumeSize(Long volumeSizeInBytes, int limitIopsPerGB) {
          if (volumeSizeInBytes == null) {
              volumeSizeInBytes = (long)ONE_GB_IN_BYTES;
          }
  
          double sizeInGB = volumeSizeInBytes / (double)ONE_GB_IN_BYTES;
  
          long limitIops = (long)(limitIopsPerGB * sizeInGB);
  
          return getAdjustedLimitIops(limitIops);
      }
  
      private long getAdjustedLimitIops(long limitIops) {
          limitIops = Math.max(limitIops, LOWEST_LIMIT_IOPS_PER_VIRTUAL_DISK);
          limitIops = Math.min(limitIops, HIGHEST_LIMIT_IOPS_PER_VIRTUAL_DISK);
  
          return limitIops;
      }
  
      private long getVolumeSizeInBytes(VolumeVO volumeVO) {
          return volumeVO.getSize() != null && volumeVO.getSize() > ONE_GB_IN_BYTES ? volumeVO.getSize() : ONE_GB_IN_BYTES;
      }
  
      private LoginInfo getLoginInfo(long zoneId) {
          VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDcZoneMapDao.findByZoneId(zoneId);
          Long associatedVmwareDcId = vmwareDcZoneMap.getVmwareDcId();
          VmwareDatacenterVO associatedVmwareDc = vmwareDcDao.findById(associatedVmwareDcId);
  
          String host = associatedVmwareDc.getVcenterHost();
          String username = associatedVmwareDc.getUser();
          String password = associatedVmwareDc.getPassword();
  
          return new LoginInfo(host, username, password);
      }
  }
  
  class ResultWrapper {
      private int limitIopsTotal;
      private List<ManagedObjectReference> tasks;
  
      ResultWrapper(int limitIopsTotal, List<ManagedObjectReference> tasks) {
          this.limitIopsTotal = limitIopsTotal;
          this.tasks = tasks;
      }
  
      int getLimitIopsTotal() {
          return limitIopsTotal;
      }
  
      List<ManagedObjectReference> getTasks() {
          return tasks;
      }
  }
  