content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.hypervisor.vmware.resource;
  
  import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
  
  import java.io.File;
  import java.io.IOException;
  import java.io.UnsupportedEncodingException;
  import java.net.ConnectException;
  import java.net.InetSocketAddress;
  import java.net.URI;
  import java.net.URL;
  import java.nio.channels.SocketChannel;
  import java.rmi.RemoteException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.Comparator;
  import java.util.Date;
  import java.util.EnumMap;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Objects;
  import java.util.Random;
  import java.util.Set;
  import java.util.TimeZone;
  import java.util.UUID;
  import java.util.stream.Collectors;
  
  import com.cloud.agent.api.CleanupVMCommand;
  import javax.naming.ConfigurationException;
  import javax.xml.datatype.XMLGregorianCalendar;
  
  import com.cloud.capacity.CapacityManager;
  import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO;
  import com.vmware.vim25.Description;
  import com.vmware.vim25.FileInfo;
  import com.vmware.vim25.FileQueryFlags;
  import com.vmware.vim25.FolderFileInfo;
  import com.vmware.vim25.HostDatastoreBrowserSearchResults;
  import com.vmware.vim25.HostDatastoreBrowserSearchSpec;
  import com.vmware.vim25.VirtualCdromIsoBackingInfo;
  import com.vmware.vim25.VirtualMachineConfigSummary;
  import com.vmware.vim25.VirtualTPM;
  import org.apache.cloudstack.api.ApiConstants;
  import org.apache.cloudstack.backup.PrepareForBackupRestorationCommand;
  import org.apache.cloudstack.storage.command.CopyCommand;
  import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
  import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer;
  import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand;
  import org.apache.cloudstack.storage.configdrive.ConfigDrive;
  import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource;
  import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
  import org.apache.cloudstack.storage.to.TemplateObjectTO;
  import org.apache.cloudstack.storage.to.VolumeObjectTO;
  import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
  import org.apache.cloudstack.vm.UnmanagedInstanceTO;
  import org.apache.commons.collections.CollectionUtils;
  import org.apache.commons.collections.MapUtils;
  import org.apache.commons.lang.ArrayUtils;
  import org.apache.commons.lang.math.NumberUtils;
  import org.apache.logging.log4j.ThreadContext;
  import org.joda.time.Duration;
  
  import com.cloud.agent.IAgentControl;
  import com.cloud.agent.api.Answer;
  import com.cloud.agent.api.AttachIsoAnswer;
  import com.cloud.agent.api.AttachIsoCommand;
  import com.cloud.agent.api.BackupSnapshotAnswer;
  import com.cloud.agent.api.BackupSnapshotCommand;
  import com.cloud.agent.api.CheckGuestOsMappingAnswer;
  import com.cloud.agent.api.CheckGuestOsMappingCommand;
  import com.cloud.agent.api.CheckHealthAnswer;
  import com.cloud.agent.api.CheckHealthCommand;
  import com.cloud.agent.api.CheckNetworkAnswer;
  import com.cloud.agent.api.CheckNetworkCommand;
  import com.cloud.agent.api.CheckOnHostAnswer;
  import com.cloud.agent.api.CheckOnHostCommand;
  import com.cloud.agent.api.CheckVirtualMachineAnswer;
  import com.cloud.agent.api.CheckVirtualMachineCommand;
  import com.cloud.agent.api.Command;
  import com.cloud.agent.api.CreatePrivateTemplateFromSnapshotCommand;
  import com.cloud.agent.api.CreatePrivateTemplateFromVolumeCommand;
  import com.cloud.agent.api.CreateStoragePoolCommand;
  import com.cloud.agent.api.CreateVMSnapshotAnswer;
  import com.cloud.agent.api.CreateVMSnapshotCommand;
  import com.cloud.agent.api.CreateVolumeFromSnapshotAnswer;
  import com.cloud.agent.api.CreateVolumeFromSnapshotCommand;
  import com.cloud.agent.api.DeleteStoragePoolCommand;
  import com.cloud.agent.api.DeleteVMSnapshotAnswer;
  import com.cloud.agent.api.DeleteVMSnapshotCommand;
  import com.cloud.agent.api.GetHostStatsAnswer;
  import com.cloud.agent.api.GetHostStatsCommand;
  import com.cloud.agent.api.GetHypervisorGuestOsNamesAnswer;
  import com.cloud.agent.api.GetHypervisorGuestOsNamesCommand;
  import com.cloud.agent.api.GetStoragePoolCapabilitiesAnswer;
  import com.cloud.agent.api.GetStoragePoolCapabilitiesCommand;
  import com.cloud.agent.api.GetStorageStatsAnswer;
  import com.cloud.agent.api.GetStorageStatsCommand;
  import com.cloud.agent.api.GetUnmanagedInstancesAnswer;
  import com.cloud.agent.api.GetUnmanagedInstancesCommand;
  import com.cloud.agent.api.GetVmDiskStatsAnswer;
  import com.cloud.agent.api.GetVmDiskStatsCommand;
  import com.cloud.agent.api.GetVmIpAddressCommand;
  import com.cloud.agent.api.GetVmNetworkStatsAnswer;
  import com.cloud.agent.api.GetVmNetworkStatsCommand;
  import com.cloud.agent.api.GetVmStatsAnswer;
  import com.cloud.agent.api.GetVmStatsCommand;
  import com.cloud.agent.api.GetVmVncTicketAnswer;
  import com.cloud.agent.api.GetVmVncTicketCommand;
  import com.cloud.agent.api.GetVncPortAnswer;
  import com.cloud.agent.api.GetVncPortCommand;
  import com.cloud.agent.api.GetVolumeStatsAnswer;
  import com.cloud.agent.api.GetVolumeStatsCommand;
  import com.cloud.agent.api.HostStatsEntry;
  import com.cloud.agent.api.HostVmStateReportEntry;
  import com.cloud.agent.api.MaintainAnswer;
  import com.cloud.agent.api.MaintainCommand;
  import com.cloud.agent.api.ManageSnapshotAnswer;
  import com.cloud.agent.api.ManageSnapshotCommand;
  import com.cloud.agent.api.MigrateAnswer;
  import com.cloud.agent.api.MigrateCommand;
  import com.cloud.agent.api.MigrateVmToPoolAnswer;
  import com.cloud.agent.api.MigrateVmToPoolCommand;
  import com.cloud.agent.api.MigrateWithStorageAnswer;
  import com.cloud.agent.api.MigrateWithStorageCommand;
  import com.cloud.agent.api.ModifySshKeysCommand;
  import com.cloud.agent.api.ModifyStoragePoolAnswer;
  import com.cloud.agent.api.ModifyStoragePoolCommand;
  import com.cloud.agent.api.ModifyTargetsAnswer;
  import com.cloud.agent.api.ModifyTargetsCommand;
  import com.cloud.agent.api.NetworkUsageAnswer;
  import com.cloud.agent.api.NetworkUsageCommand;
  import com.cloud.agent.api.PatchSystemVmAnswer;
  import com.cloud.agent.api.PatchSystemVmCommand;
  import com.cloud.agent.api.PingCommand;
  import com.cloud.agent.api.PingRoutingCommand;
  import com.cloud.agent.api.PingTestCommand;
  import com.cloud.agent.api.PlugNicAnswer;
  import com.cloud.agent.api.PlugNicCommand;
  import com.cloud.agent.api.PrepareForMigrationAnswer;
  import com.cloud.agent.api.PrepareForMigrationCommand;
  import com.cloud.agent.api.PrepareUnmanageVMInstanceAnswer;
  import com.cloud.agent.api.PrepareUnmanageVMInstanceCommand;
  import com.cloud.agent.api.PvlanSetupCommand;
  import com.cloud.agent.api.ReadyAnswer;
  import com.cloud.agent.api.ReadyCommand;
  import com.cloud.agent.api.RebootAnswer;
  import com.cloud.agent.api.RebootCommand;
  import com.cloud.agent.api.RebootRouterCommand;
  import com.cloud.agent.api.ReplugNicAnswer;
  import com.cloud.agent.api.ReplugNicCommand;
  import com.cloud.agent.api.RevertToVMSnapshotAnswer;
  import com.cloud.agent.api.RevertToVMSnapshotCommand;
  import com.cloud.agent.api.ScaleVmAnswer;
  import com.cloud.agent.api.ScaleVmCommand;
  import com.cloud.agent.api.SetupAnswer;
  import com.cloud.agent.api.SetupCommand;
  import com.cloud.agent.api.SetupGuestNetworkCommand;
  import com.cloud.agent.api.SetupPersistentNetworkAnswer;
  import com.cloud.agent.api.SetupPersistentNetworkCommand;
  import com.cloud.agent.api.StartAnswer;
  import com.cloud.agent.api.StartCommand;
  import com.cloud.agent.api.StartupCommand;
  import com.cloud.agent.api.StartupRoutingCommand;
  import com.cloud.agent.api.StartupStorageCommand;
  import com.cloud.agent.api.StopAnswer;
  import com.cloud.agent.api.StopCommand;
  import com.cloud.agent.api.StoragePoolInfo;
  import com.cloud.agent.api.UnPlugNicAnswer;
  import com.cloud.agent.api.UnPlugNicCommand;
  import com.cloud.agent.api.UnregisterNicCommand;
  import com.cloud.agent.api.UnregisterVMCommand;
  import com.cloud.agent.api.UpgradeSnapshotCommand;
  import com.cloud.agent.api.ValidateSnapshotAnswer;
  import com.cloud.agent.api.ValidateSnapshotCommand;
  import com.cloud.agent.api.ValidateVcenterDetailsCommand;
  import com.cloud.agent.api.VmDiskStatsEntry;
  import com.cloud.agent.api.VmStatsEntry;
  import com.cloud.agent.api.VolumeStatsEntry;
  import com.cloud.agent.api.check.CheckSshAnswer;
  import com.cloud.agent.api.check.CheckSshCommand;
  import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer;
  import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand;
  import com.cloud.agent.api.routing.IpAssocCommand;
  import com.cloud.agent.api.routing.IpAssocVpcCommand;
  import com.cloud.agent.api.routing.NetworkElementCommand;
  import com.cloud.agent.api.routing.SetNetworkACLCommand;
  import com.cloud.agent.api.routing.SetSourceNatCommand;
  import com.cloud.agent.api.storage.CopyVolumeAnswer;
  import com.cloud.agent.api.storage.CopyVolumeCommand;
  import com.cloud.agent.api.storage.CreatePrivateTemplateAnswer;
  import com.cloud.agent.api.storage.DestroyCommand;
  import com.cloud.agent.api.storage.MigrateVolumeAnswer;
  import com.cloud.agent.api.storage.MigrateVolumeCommand;
  import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer;
  import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
  import com.cloud.agent.api.storage.ResizeVolumeAnswer;
  import com.cloud.agent.api.storage.ResizeVolumeCommand;
  import com.cloud.agent.api.to.DataStoreTO;
  import com.cloud.agent.api.to.DataTO;
  import com.cloud.agent.api.to.DeployAsIsInfoTO;
  import com.cloud.agent.api.to.DiskTO;
  import com.cloud.agent.api.to.IpAddressTO;
  import com.cloud.agent.api.to.NfsTO;
  import com.cloud.agent.api.to.NicTO;
  import com.cloud.agent.api.to.StorageFilerTO;
  import com.cloud.agent.api.to.VirtualMachineTO;
  import com.cloud.agent.api.to.VolumeTO;
  import com.cloud.agent.api.to.deployasis.OVFPropertyTO;
  import com.cloud.agent.resource.virtualnetwork.VRScripts;
  import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer;
  import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource;
  import com.cloud.configuration.Resource.ResourceType;
  import com.cloud.dc.Vlan;
  import com.cloud.exception.CloudException;
  import com.cloud.exception.InternalErrorException;
  import com.cloud.host.Host;
  import com.cloud.host.Host.Type;
  import com.cloud.hypervisor.Hypervisor.HypervisorType;
  import com.cloud.hypervisor.guru.VMwareGuru;
  import com.cloud.hypervisor.vmware.manager.VmwareHostService;
  import com.cloud.hypervisor.vmware.manager.VmwareManager;
  import com.cloud.hypervisor.vmware.manager.VmwareStorageMount;
  import com.cloud.hypervisor.vmware.mo.ClusterMO;
  import com.cloud.hypervisor.vmware.mo.CustomFieldConstants;
  import com.cloud.hypervisor.vmware.mo.CustomFieldsManagerMO;
  import com.cloud.hypervisor.vmware.mo.DatacenterMO;
  import com.cloud.hypervisor.vmware.mo.DatastoreFile;
  import com.cloud.hypervisor.vmware.mo.DatastoreMO;
  import com.cloud.hypervisor.vmware.mo.DiskControllerType;
  import com.cloud.hypervisor.vmware.mo.FeatureKeyConstants;
  import com.cloud.hypervisor.vmware.mo.HostDatastoreSystemMO;
  import com.cloud.hypervisor.vmware.mo.HostMO;
  import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO;
  import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
  import com.cloud.hypervisor.vmware.mo.NetworkDetails;
  import com.cloud.hypervisor.vmware.mo.NetworkMO;
  import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO;
  import com.cloud.hypervisor.vmware.mo.StoragepodMO;
  import com.cloud.hypervisor.vmware.mo.TaskMO;
  import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType;
  import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder;
  import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
  import com.cloud.hypervisor.vmware.mo.VirtualSwitchType;
  import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost;
  import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary;
  import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostResourceSummary;
  import com.cloud.hypervisor.vmware.util.VmwareContext;
  import com.cloud.hypervisor.vmware.util.VmwareContextPool;
  import com.cloud.hypervisor.vmware.util.VmwareHelper;
  import com.cloud.network.Networks;
  import com.cloud.network.Networks.BroadcastDomainType;
  import com.cloud.network.Networks.TrafficType;
  import com.cloud.network.VmwareTrafficLabel;
  import com.cloud.network.router.VirtualRouterAutoScale;
  import com.cloud.resource.ServerResource;
  import com.cloud.resource.ServerResourceBase;
  import com.cloud.serializer.GsonHelper;
  import com.cloud.storage.Storage;
  import com.cloud.storage.Storage.StoragePoolType;
  import com.cloud.storage.Volume;
  import com.cloud.storage.resource.StoragePoolResource;
  import com.cloud.storage.resource.StorageSubsystemCommandHandler;
  import com.cloud.storage.resource.VmwareStorageLayoutHelper;
  import com.cloud.storage.resource.VmwareStorageProcessor;
  import com.cloud.storage.resource.VmwareStorageProcessor.VmwareStorageProcessorConfigurableFields;
  import com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler;
  import com.cloud.storage.template.TemplateProp;
  import com.cloud.template.TemplateManager;
  import com.cloud.utils.DateUtil;
  import com.cloud.utils.ExecutionResult;
  import com.cloud.utils.FileUtil;
  import com.cloud.utils.LogUtils;
  import com.cloud.utils.NumbersUtil;
  import com.cloud.utils.Pair;
  import com.cloud.utils.StringUtils;
  import com.cloud.utils.Ternary;
  import com.cloud.utils.UuidUtils;
  import com.cloud.utils.db.DB;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.exception.ExceptionUtil;
  import com.cloud.utils.mgmt.JmxUtil;
  import com.cloud.utils.mgmt.PropertyMapDynamicBean;
  import com.cloud.utils.net.NetUtils;
  import com.cloud.utils.nicira.nvp.plugin.NiciraNvpApiVersion;
  import com.cloud.utils.script.Script;
  import com.cloud.utils.ssh.SshHelper;
  import com.cloud.utils.validation.ChecksumUtil;
  import com.cloud.vm.VirtualMachine;
  import com.cloud.vm.VirtualMachine.PowerState;
  import com.cloud.vm.VirtualMachineName;
  import com.cloud.vm.VmDetailConstants;
  import com.google.gson.Gson;
  import com.vmware.vim25.AboutInfo;
  import com.vmware.vim25.ArrayUpdateOperation;
  import com.vmware.vim25.BoolPolicy;
  import com.vmware.vim25.ComputeResourceSummary;
  import com.vmware.vim25.CustomFieldStringValue;
  import com.vmware.vim25.DVPortConfigInfo;
  import com.vmware.vim25.DVPortConfigSpec;
  import com.vmware.vim25.DasVmPriority;
  import com.vmware.vim25.DatastoreSummary;
  import com.vmware.vim25.DistributedVirtualPort;
  import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
  import com.vmware.vim25.DistributedVirtualSwitchPortCriteria;
  import com.vmware.vim25.DynamicProperty;
  import com.vmware.vim25.GuestInfo;
  import com.vmware.vim25.GuestOsDescriptor;
  import com.vmware.vim25.HostCapability;
  import com.vmware.vim25.HostConfigInfo;
  import com.vmware.vim25.HostFileSystemMountInfo;
  import com.vmware.vim25.HostHostBusAdapter;
  import com.vmware.vim25.HostInternetScsiHba;
  import com.vmware.vim25.ManagedObjectReference;
  import com.vmware.vim25.ObjectContent;
  import com.vmware.vim25.OptionValue;
  import com.vmware.vim25.PerfCounterInfo;
  import com.vmware.vim25.PerfEntityMetric;
  import com.vmware.vim25.PerfEntityMetricBase;
  import com.vmware.vim25.PerfMetricId;
  import com.vmware.vim25.PerfMetricIntSeries;
  import com.vmware.vim25.PerfMetricSeries;
  import com.vmware.vim25.PerfQuerySpec;
  import com.vmware.vim25.RuntimeFaultFaultMsg;
  import com.vmware.vim25.StoragePodSummary;
  import com.vmware.vim25.ToolsUnavailableFaultMsg;
  import com.vmware.vim25.VAppOvfSectionInfo;
  import com.vmware.vim25.VAppOvfSectionSpec;
  import com.vmware.vim25.VAppProductInfo;
  import com.vmware.vim25.VAppProductSpec;
  import com.vmware.vim25.VAppPropertyInfo;
  import com.vmware.vim25.VAppPropertySpec;
  import com.vmware.vim25.VMwareDVSPortSetting;
  import com.vmware.vim25.VimPortType;
  import com.vmware.vim25.VirtualDevice;
  import com.vmware.vim25.VirtualDeviceBackingInfo;
  import com.vmware.vim25.VirtualDeviceConfigSpec;
  import com.vmware.vim25.VirtualDeviceConfigSpecOperation;
  import com.vmware.vim25.VirtualDisk;
  import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo;
  import com.vmware.vim25.VirtualEthernetCard;
  import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
  import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
  import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo;
  import com.vmware.vim25.VirtualMachineBootOptions;
  import com.vmware.vim25.VirtualMachineConfigSpec;
  import com.vmware.vim25.VirtualMachineDefinedProfileSpec;
  import com.vmware.vim25.VirtualMachineFileInfo;
  import com.vmware.vim25.VirtualMachineFileLayoutEx;
  import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo;
  import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
  import com.vmware.vim25.VirtualMachinePowerState;
  import com.vmware.vim25.VirtualMachineRelocateSpec;
  import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator;
  import com.vmware.vim25.VirtualMachineRuntimeInfo;
  import com.vmware.vim25.VirtualMachineToolsStatus;
  import com.vmware.vim25.VirtualMachineVideoCard;
  import com.vmware.vim25.VirtualSCSIController;
  import com.vmware.vim25.VirtualUSBController;
  import com.vmware.vim25.VmConfigInfo;
  import com.vmware.vim25.VmConfigSpec;
  import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec;
  
  public class VmwareResource extends ServerResourceBase implements StoragePoolResource, ServerResource, VmwareHostService, VirtualRouterDeployer {
      public static final String VMDK_EXTENSION = ".vmdk";
      private static final String EXECUTING_RESOURCE_COMMAND = "Executing resource command %s: [%s].";
      public static final String BASEPATH = "/usr/share/cloudstack-common/vms/";
  
      private static final Random RANDOM = new Random(System.nanoTime());
  
      protected String _name;
  
      protected final long _opsTimeout = 900000;   // 15 minutes time out to time
  
      protected final int _shutdownWaitMs = 300000;  // wait up to 5 minutes for shutdown
  
      // out an operation
      protected final int _retry = 24;
      protected final int DefaultDomRSshPort = 3922;
      protected final int MazCmdMBean = 100;
  
      protected String _url;
      protected String _dcId;
      protected String _pod;
      protected String _cluster;
      protected String _username;
      protected String _password;
      protected String _guid;
      protected String _vCenterAddress;
      protected String storageNfsVersion;
  
      protected String _privateNetworkVSwitchName;
      protected VmwareTrafficLabel _guestTrafficInfo = new VmwareTrafficLabel(TrafficType.Guest);
      protected VmwareTrafficLabel _publicTrafficInfo = new VmwareTrafficLabel(TrafficType.Public);
      protected Map<String, String> _vsmCredentials = null;
      protected int _portsPerDvPortGroup;
      protected boolean _fullCloneFlag = false;
      protected boolean _instanceNameFlag = false;
  
      protected boolean _recycleHungWorker = false;
      protected DiskControllerType _rootDiskController = DiskControllerType.ide;
  
      protected ManagedObjectReference _morHyperHost;
      protected final static ThreadLocal<VmwareContext> s_serviceContext = new ThreadLocal<>();
      protected String _hostName;
  
      protected List<PropertyMapDynamicBean> _cmdMBeans = new ArrayList<>();
  
      protected Gson _gson;
  
      protected volatile long _cmdSequence = 1;
  
      protected StorageSubsystemCommandHandler storageHandler;
      private VmwareStorageProcessor _storageProcessor;
  
      protected VirtualRoutingResource _vrResource;
  
      protected final static HashMap<VirtualMachinePowerState, PowerState> s_powerStatesTable = new HashMap<>();
  
      static {
          s_powerStatesTable.put(VirtualMachinePowerState.POWERED_ON, PowerState.PowerOn);
          s_powerStatesTable.put(VirtualMachinePowerState.POWERED_OFF, PowerState.PowerOff);
          s_powerStatesTable.put(VirtualMachinePowerState.SUSPENDED, PowerState.PowerOn);
      }
  
      protected static File s_systemVmKeyFile = null;
      private static final Object s_syncLockObjectFetchKeyFile = new Object();
      protected static final String s_relativePathSystemVmKeyFileInstallDir = "scripts/vm/systemvm/id_rsa.cloud";
      protected static final String s_defaultPathSystemVmKeyFile = "/usr/share/cloudstack-common/scripts/vm/systemvm/id_rsa.cloud";
  
      public VmwareResource() {
          _gson = GsonHelper.getGsonLogger();
      }
  
      private String getCommandLogTitle(Command cmd) {
          StringBuffer sb = new StringBuffer();
          if (_hostName != null) {
              sb.append(_hostName);
          }
  
          if (cmd.getContextParam("job") != null) {
              sb.append(", ").append(cmd.getContextParam("job"));
          }
          sb.append(", cmd: ").append(cmd.getClass().getSimpleName());
  
          return sb.toString();
      }
  
      @Override
      public Answer executeRequest(Command cmd) {
          logCommand(cmd);
          Answer answer;
          ThreadContext.push(getCommandLogTitle(cmd));
          try {
              long cmdSequence = _cmdSequence++;
              Date startTime = DateUtil.currentGMTTime();
              PropertyMapDynamicBean mbean = new PropertyMapDynamicBean();
              mbean.addProp("StartTime", DateUtil.getDateDisplayString(TimeZone.getDefault(), startTime));
              mbean.addProp("Command", _gson.toJson(cmd));
              mbean.addProp("Sequence", String.valueOf(cmdSequence));
              mbean.addProp("Name", cmd.getClass().getSimpleName());
  
              Class<? extends Command> clz = cmd.getClass();
              if (clz == PatchSystemVmCommand.class) {
                  answer = execute((PatchSystemVmCommand) cmd);
              } else if (cmd instanceof NetworkElementCommand) {
                  return _vrResource.executeRequest((NetworkElementCommand) cmd);
              } else if (clz == ReadyCommand.class) {
                  answer = execute((ReadyCommand) cmd);
              } else if (clz == GetHostStatsCommand.class) {
                  answer = execute((GetHostStatsCommand) cmd);
              } else if (clz == GetVmStatsCommand.class) {
                  answer = execute((GetVmStatsCommand) cmd);
              } else if (clz == GetVmNetworkStatsCommand.class) {
                  answer = execute((GetVmNetworkStatsCommand) cmd);
              } else if (clz == GetVmDiskStatsCommand.class) {
                  answer = execute((GetVmDiskStatsCommand) cmd);
              } else if (cmd instanceof GetVolumeStatsCommand) {
                  return execute((GetVolumeStatsCommand) cmd);
              } else if (clz == CheckHealthCommand.class) {
                  answer = execute((CheckHealthCommand) cmd);
              } else if (clz == StopCommand.class) {
                  answer = execute((StopCommand) cmd);
              } else if (clz == RebootRouterCommand.class) {
                  answer = execute((RebootRouterCommand) cmd);
              } else if (clz == RebootCommand.class) {
                  answer = execute((RebootCommand) cmd);
              } else if (clz == CheckVirtualMachineCommand.class) {
                  answer = execute((CheckVirtualMachineCommand) cmd);
              } else if (clz == PrepareForMigrationCommand.class) {
                  answer = execute((PrepareForMigrationCommand) cmd);
              } else if (clz == MigrateCommand.class) {
                  answer = execute((MigrateCommand) cmd);
              } else if (clz == MigrateVmToPoolCommand.class) {
                  answer = execute((MigrateVmToPoolCommand) cmd);
              } else if (clz == MigrateWithStorageCommand.class) {
                  answer = execute((MigrateWithStorageCommand) cmd);
              } else if (clz == MigrateVolumeCommand.class) {
                  answer = execute((MigrateVolumeCommand) cmd);
              } else if (clz == DestroyCommand.class) {
                  answer = execute((DestroyCommand) cmd);
              } else if (clz == CreateStoragePoolCommand.class) {
                  return execute((CreateStoragePoolCommand) cmd);
              } else if (clz == ModifyTargetsCommand.class) {
                  answer = execute((ModifyTargetsCommand) cmd);
              } else if (clz == ModifyStoragePoolCommand.class) {
                  answer = execute((ModifyStoragePoolCommand) cmd);
              } else if (clz == GetStoragePoolCapabilitiesCommand.class) {
                  answer = execute((GetStoragePoolCapabilitiesCommand) cmd);
              } else if (clz == DeleteStoragePoolCommand.class) {
                  answer = execute((DeleteStoragePoolCommand) cmd);
              } else if (clz == CopyVolumeCommand.class) {
                  answer = execute((CopyVolumeCommand) cmd);
              } else if (clz == AttachIsoCommand.class) {
                  answer = execute((AttachIsoCommand) cmd);
              } else if (clz == ValidateSnapshotCommand.class) {
                  answer = execute((ValidateSnapshotCommand) cmd);
              } else if (clz == ManageSnapshotCommand.class) {
                  answer = execute((ManageSnapshotCommand) cmd);
              } else if (clz == BackupSnapshotCommand.class) {
                  answer = execute((BackupSnapshotCommand) cmd);
              } else if (clz == CreateVolumeFromSnapshotCommand.class) {
                  answer = execute((CreateVolumeFromSnapshotCommand) cmd);
              } else if (clz == CreatePrivateTemplateFromVolumeCommand.class) {
                  answer = execute((CreatePrivateTemplateFromVolumeCommand) cmd);
              } else if (clz == CreatePrivateTemplateFromSnapshotCommand.class) {
                  answer = execute((CreatePrivateTemplateFromSnapshotCommand) cmd);
              } else if (clz == UpgradeSnapshotCommand.class) {
                  answer = execute((UpgradeSnapshotCommand) cmd);
              } else if (clz == GetStorageStatsCommand.class) {
                  answer = execute((GetStorageStatsCommand) cmd);
              } else if (clz == PrimaryStorageDownloadCommand.class) {
                  answer = execute((PrimaryStorageDownloadCommand) cmd);
              } else if (clz == GetVncPortCommand.class) {
                  answer = execute((GetVncPortCommand) cmd);
              } else if (clz == SetupCommand.class) {
                  answer = execute((SetupCommand) cmd);
              } else if (clz == MaintainCommand.class) {
                  answer = execute((MaintainCommand) cmd);
              } else if (clz == PingTestCommand.class) {
                  answer = execute((PingTestCommand) cmd);
              } else if (clz == CheckOnHostCommand.class) {
                  answer = execute((CheckOnHostCommand) cmd);
              } else if (clz == ModifySshKeysCommand.class) {
                  answer = execute((ModifySshKeysCommand) cmd);
              } else if (clz == NetworkUsageCommand.class) {
                  answer = execute((NetworkUsageCommand) cmd);
              } else if (clz == StartCommand.class) {
                  answer = execute((StartCommand) cmd);
              } else if (clz == CheckSshCommand.class) {
                  answer = execute((CheckSshCommand) cmd);
              } else if (clz == CheckNetworkCommand.class) {
                  answer = execute((CheckNetworkCommand) cmd);
              } else if (clz == PlugNicCommand.class) {
                  answer = execute((PlugNicCommand) cmd);
              } else if (clz == ReplugNicCommand.class) {
                  answer = execute((ReplugNicCommand) cmd);
              } else if (clz == UnPlugNicCommand.class) {
                  answer = execute((UnPlugNicCommand) cmd);
              } else if (cmd instanceof CreateVMSnapshotCommand) {
                  return execute((CreateVMSnapshotCommand) cmd);
              } else if (cmd instanceof DeleteVMSnapshotCommand) {
                  return execute((DeleteVMSnapshotCommand) cmd);
              } else if (cmd instanceof RevertToVMSnapshotCommand) {
                  return execute((RevertToVMSnapshotCommand) cmd);
              } else if (clz == ResizeVolumeCommand.class) {
                  return execute((ResizeVolumeCommand) cmd);
              } else if (clz == UnregisterVMCommand.class) {
                  return execute((UnregisterVMCommand) cmd);
              } else if (clz == CleanupVMCommand.class) {
                  return execute((CleanupVMCommand) cmd);
              } else if (cmd instanceof StorageSubSystemCommand) {
                  checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd);
                  return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd);
              } else if (clz == ScaleVmCommand.class) {
                  return execute((ScaleVmCommand) cmd);
              } else if (clz == PvlanSetupCommand.class) {
                  return execute((PvlanSetupCommand) cmd);
              } else if (clz == GetVmIpAddressCommand.class) {
                  return execute((GetVmIpAddressCommand) cmd);
              } else if (clz == UnregisterNicCommand.class) {
                  answer = execute((UnregisterNicCommand) cmd);
              } else if (clz == GetUnmanagedInstancesCommand.class) {
                  answer = execute((GetUnmanagedInstancesCommand) cmd);
              } else if (clz == PrepareUnmanageVMInstanceCommand.class) {
                  answer = execute((PrepareUnmanageVMInstanceCommand) cmd);
              } else if (clz == ValidateVcenterDetailsCommand.class) {
                  answer = execute((ValidateVcenterDetailsCommand) cmd);
              } else if (clz == SetupPersistentNetworkCommand.class) {
                  answer = execute((SetupPersistentNetworkCommand) cmd);
              } else if (clz == GetVmVncTicketCommand.class) {
                  answer = execute((GetVmVncTicketCommand) cmd);
              } else if (clz == GetAutoScaleMetricsCommand.class) {
                  answer = execute((GetAutoScaleMetricsCommand) cmd);
              } else if (clz == CheckGuestOsMappingCommand.class) {
                  answer = execute((CheckGuestOsMappingCommand) cmd);
              } else if (clz == GetHypervisorGuestOsNamesCommand.class) {
                  answer = execute((GetHypervisorGuestOsNamesCommand) cmd);
              } else if (clz == ListDataStoreObjectsCommand.class) {
                  answer = execute((ListDataStoreObjectsCommand) cmd);
              } else if (clz == PrepareForBackupRestorationCommand.class) {
                  answer = execute((PrepareForBackupRestorationCommand) cmd);
              } else {
                  answer = Answer.createUnsupportedCommandAnswer(cmd);
              }
  
              if (cmd.getContextParam("checkpoint") != null) {
                  answer.setContextParam("checkpoint", cmd.getContextParam("checkpoint"));
              }
  
              Date doneTime = DateUtil.currentGMTTime();
              mbean.addProp("DoneTime", DateUtil.getDateDisplayString(TimeZone.getDefault(), doneTime));
              mbean.addProp("Answer", _gson.toJson(answer));
  
              synchronized (this) {
                  try {
                      JmxUtil.registerMBean("VMware " + _morHyperHost.getValue(), "Command " + cmdSequence + "-" + cmd.getClass().getSimpleName(), mbean);
                      _cmdMBeans.add(mbean);
  
                      if (_cmdMBeans.size() >= MazCmdMBean) {
                          PropertyMapDynamicBean mbeanToRemove = _cmdMBeans.get(0);
                          _cmdMBeans.remove(0);
  
                          JmxUtil.unregisterMBean("VMware " + _morHyperHost.getValue(), "Command " + mbeanToRemove.getProp("Sequence") + "-" + mbeanToRemove.getProp("Name"));
                      }
                  } catch (Exception e) {
                      if (logger.isTraceEnabled())
                          logger.trace("Unable to register JMX monitoring due to exception " + ExceptionUtil.toString(e));
                  }
              }
  
          } finally {
              recycleServiceContext();
              ThreadContext.pop();
          }
  
          if (logger.isTraceEnabled())
              logger.trace("End executeRequest(), cmd: " + cmd.getClass().getSimpleName());
  
          return answer;
      }
  
      private ExecutionResult getSystemVmVersionAndChecksum(String controlIp) {
          ExecutionResult result;
          try {
              result = executeInVR(controlIp, VRScripts.VERSION, null);
              if (!result.isSuccess()) {
                  String errMsg = String.format("GetSystemVMVersionCmd on %s failed, message %s", controlIp, result.getDetails());
                  logger.error(errMsg);
                  throw new CloudRuntimeException(errMsg);
              }
          } catch (final Exception e) {
              final String msg = "GetSystemVMVersionCmd failed due to " + e;
              logger.error(msg, e);
              throw new CloudRuntimeException(msg, e);
          }
          return result;
      }
  
      private Answer execute(PatchSystemVmCommand cmd) {
          String controlIp = cmd.getAccessDetail((NetworkElementCommand.ROUTER_IP));
          String sysVMName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
          String homeDir = System.getProperty("user.home");
          File pemFile = new File(homeDir + "/.ssh/id_rsa");
          ExecutionResult result;
          try {
              result = getSystemVmVersionAndChecksum(controlIp);
              FileUtil.scpPatchFiles(controlIp, VRScripts.CONFIG_CACHE_LOCATION, DefaultDomRSshPort, pemFile, systemVmPatchFiles, BASEPATH);
          } catch (CloudRuntimeException e) {
              return new PatchSystemVmAnswer(cmd, e.getMessage());
          }
  
          final String[] lines = result.getDetails().split("&");
          // TODO: do we fail, or patch anyway??
          if (lines.length != 2) {
              return new PatchSystemVmAnswer(cmd, result.getDetails());
          }
  
          String scriptChecksum = lines[1].trim();
          String checksum = ChecksumUtil.calculateCurrentChecksum(sysVMName, "vms/cloud-scripts.tgz").trim();
  
          if (!StringUtils.isEmpty(checksum) && checksum.equals(scriptChecksum) && !cmd.isForced()) {
              String msg = String.format("No change in the scripts checksum, not patching systemVM %s", sysVMName);
              logger.info(msg);
              return new PatchSystemVmAnswer(cmd, msg, lines[0], lines[1]);
          }
  
          Pair<Boolean, String> patchResult;
          try {
              patchResult = SshHelper.sshExecute(controlIp, DefaultDomRSshPort, "root",
                      pemFile, null, "/var/cache/cloud/patch-sysvms.sh", 10000, 10000, 600000);
          } catch (Exception e) {
              return new PatchSystemVmAnswer(cmd, e.getMessage());
          }
  
          String scriptVersion = lines[1];
          if (StringUtils.isNotEmpty(patchResult.second())) {
              String res = patchResult.second().replace("\n", " ");
              String[] output = res.split(":");
              if (output.length != 2) {
                  logger.warn("Failed to get the latest script version");
              } else {
                  scriptVersion = output[1].split(" ")[0];
              }
  
          }
          if (patchResult.first()) {
              return new PatchSystemVmAnswer(cmd, String.format("Successfully patched systemVM %s ", sysVMName), lines[0], scriptVersion);
          }
          return new PatchSystemVmAnswer(cmd, patchResult.second());
  
      }
  
      private Answer execute(SetupPersistentNetworkCommand cmd) {
          VmwareHypervisorHost host = getHyperHost(getServiceContext());
          String hostname;
          VmwareContext context = getServiceContext();
          HostMO hostMO = new HostMO(context, host.getMor());
  
          try {
              prepareNetworkFromNicInfo(hostMO, cmd.getNic(), false, null, null);
              hostname =  host.getHyperHostName();
          } catch (Exception e) {
              return new SetupPersistentNetworkAnswer(cmd, false, "failed to setup port-group due to: "+ e.getLocalizedMessage());
          }
          return new SetupPersistentNetworkAnswer(cmd, true, hostname);
      }
  
      /**
       * Check if storage NFS version is already set or needs to be reconfigured.<br>
       * If _storageNfsVersion is not null -> nothing to do, version already set.<br>
       * If _storageNfsVersion is null -> examine StorageSubSystemCommand to get NFS version and set it
       * to the storage processor and storage handler.
       *
       * @param cmd command to execute
       */
      protected void checkStorageProcessorAndHandlerNfsVersionAttribute(StorageSubSystemCommand cmd) {
          if (storageNfsVersion != null)
              return;
          if (cmd instanceof CopyCommand) {
              EnumMap<VmwareStorageProcessorConfigurableFields, Object> params = new EnumMap<>(
                      VmwareStorageProcessorConfigurableFields.class);
              examineStorageSubSystemCommandNfsVersion((CopyCommand) cmd, params);
              params = examineStorageSubSystemCommandFullCloneFlagForVmware((CopyCommand) cmd, params);
              reconfigureProcessorByHandler(params);
          }
      }
  
      /**
       * Reconfigure processor by handler
       *
       * @param params params
       */
      protected void reconfigureProcessorByHandler(EnumMap<VmwareStorageProcessorConfigurableFields, Object> params) {
          VmwareStorageSubsystemCommandHandler handler = (VmwareStorageSubsystemCommandHandler) storageHandler;
          boolean success = handler.reconfigureStorageProcessor(params);
          if (success) {
              logger.info("VmwareStorageProcessor and VmwareStorageSubsystemCommandHandler successfully reconfigured");
          } else {
              logger.error("Error while reconfiguring VmwareStorageProcessor and VmwareStorageSubsystemCommandHandler, params=" + _gson.toJson(params));
          }
      }
  
      /**
       * Examine StorageSubSystem command to get full clone flag, if provided
       *
       * @param cmd    command to execute
       * @param params params
       * @return copy of params including new values, if suitable
       */
      protected EnumMap<VmwareStorageProcessorConfigurableFields, Object> examineStorageSubSystemCommandFullCloneFlagForVmware(CopyCommand cmd,
              EnumMap<VmwareStorageProcessorConfigurableFields, Object> params) {
          EnumMap<VmwareStorageProcessorConfigurableFields, Object> paramsCopy = new EnumMap<>(params);
          HypervisorType hypervisor = cmd.getDestTO().getHypervisorType();
          if (hypervisor != null && hypervisor.equals(HypervisorType.VMware)) {
              DataStoreTO destDataStore = cmd.getDestTO().getDataStore();
              if (destDataStore instanceof PrimaryDataStoreTO) {
                  PrimaryDataStoreTO dest = (PrimaryDataStoreTO) destDataStore;
                  if (dest.isFullCloneFlag() != null) {
                      paramsCopy.put(VmwareStorageProcessorConfigurableFields.FULL_CLONE_FLAG, dest.isFullCloneFlag());
                  }
                  if (dest.getDiskProvisioningStrictnessFlag() != null) {
                      paramsCopy.put(VmwareStorageProcessorConfigurableFields.DISK_PROVISIONING_STRICTNESS, dest.getDiskProvisioningStrictnessFlag());
                  }
              }
          }
          return paramsCopy;
      }
  
      /**
       * Examine StorageSubSystem command to get storage NFS version, if provided
       *
       * @param cmd    command to execute
       * @param params params
       */
      protected void examineStorageSubSystemCommandNfsVersion(CopyCommand cmd, EnumMap<VmwareStorageProcessorConfigurableFields, Object> params) {
          DataStoreTO srcDataStore = cmd.getSrcTO().getDataStore();
          boolean nfsVersionFound = false;
  
          if (srcDataStore instanceof NfsTO) {
              nfsVersionFound = getStorageNfsVersionFromNfsTO((NfsTO) srcDataStore);
          }
  
          if (nfsVersionFound) {
              params.put(VmwareStorageProcessorConfigurableFields.NFS_VERSION, storageNfsVersion);
          }
      }
  
      /**
       * Get storage NFS version from NfsTO
       *
       * @param nfsTO nfsTO
       * @return true if NFS version was found and not null, false in other case
       */
      protected boolean getStorageNfsVersionFromNfsTO(NfsTO nfsTO) {
          if (nfsTO != null && nfsTO.getNfsVersion() != null) {
              storageNfsVersion = nfsTO.getNfsVersion();
              return true;
          }
          return false;
      }
  
      /**
       * Registers the vm to the inventory given the vmx file.
       */
      private void registerVm(String vmName, DatastoreMO dsMo) throws Exception {
  
          //1st param
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
          DatacenterMO dataCenterMo = new DatacenterMO(getServiceContext(), dcMor);
          ManagedObjectReference vmFolderMor = dataCenterMo.getVmFolder();
  
          //2nd param
          String vmxFilePath = dsMo.searchFileInSubFolders(vmName + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value());
  
          // 5th param
          ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool();
  
          ManagedObjectReference morTask = getServiceContext().getService().registerVMTask(vmFolderMor, vmxFilePath, vmName, false, morPool, hyperHost.getMor());
          boolean result = getServiceContext().getVimClient().waitForTask(morTask);
          if (!result) {
              throw new Exception("Unable to register vm due to " + TaskMO.getTaskFailureInfo(getServiceContext(), morTask));
          } else {
              getServiceContext().waitForTaskProgressDone(morTask);
          }
  
      }
  
      private Answer execute(ResizeVolumeCommand cmd) {
          String path = cmd.getPath();
          String vmName = cmd.getInstanceName();
          long newSize = cmd.getNewSize() / ResourceType.bytesToKiB;
          long oldSize = cmd.getCurrentSize() / ResourceType.bytesToKiB;
          boolean managed = cmd.isManaged();
          String poolUUID = cmd.getPoolUuid();
          String chainInfo = cmd.getChainInfo();
          boolean useWorkerVm = false;
  
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          VirtualMachineMO vmMo = null;
  
          String vmdkDataStorePath = null;
  
          try {
              if (newSize < oldSize) {
                  String errorMsg = String.format("VMware doesn't support shrinking volume from larger size [%s] GB to a smaller size [%s] GB. Can't resize volume of VM [name: %s].",
                          oldSize / (float) ResourceType.bytesToMiB, newSize / (float) ResourceType.bytesToMiB, vmName);
                  logger.error(errorMsg);
                  throw new Exception(errorMsg);
              } else if (newSize == oldSize) {
                  return new ResizeVolumeAnswer(cmd, true, "success", newSize * ResourceType.bytesToKiB);
              }
  
              if (vmName.equalsIgnoreCase("none")) {
                  // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here
                  // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway
                  // we need to spawn a worker VM to attach the volume to and resize the volume.
                  useWorkerVm = true;
  
                  String poolId = cmd.getPoolUuid();
  
                  // OfflineVmwareMigration: refactor for re-use
                  // OfflineVmwareMigration: 1. find data(store)
                  ManagedObjectReference morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId);
                  DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDS);
                  vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo);
  
                  logger.info("Create worker VM " + vmName);
  
                  // OfflineVmwareMigration: 2. create the worker with access to the data(store)
                  vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, vmName, null);
  
                  if (vmMo == null) {
                      // OfflineVmwareMigration: don't throw a general Exception but think of a specific one
                      throw new Exception("Unable to create a worker VM for volume resize");
                  }
  
                  synchronized (this) {
                      // OfflineVmwareMigration: 3. attach the disk to the worker
                      vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, path + VMDK_EXTENSION);
  
                      vmMo.attachDisk(new String[]{vmdkDataStorePath}, morDS);
                  }
              }
  
              // OfflineVmwareMigration: 4. find the (worker-) VM
              // find VM through datacenter (VM is not at the target host yet)
              vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
  
              if (vmMo == null) {
                  String errorMsg = String.format("VM [name: %s] does not exist in VMware datacenter.", vmName);
                  logger.error(errorMsg);
                  throw new Exception(errorMsg);
              }
  
  
              if (managed) {
                  ManagedObjectReference morCluster = hyperHost.getHyperHostCluster();
                  ClusterMO clusterMO = new ClusterMO(context, morCluster);
  
                  List<Pair<ManagedObjectReference, String>> lstHosts = clusterMO.getClusterHosts();
  
                  Collections.shuffle(lstHosts, RANDOM);
  
                  Pair<ManagedObjectReference, String> host = lstHosts.get(0);
  
                  HostMO hostMO = new HostMO(context, host.first());
                  HostDatastoreSystemMO hostDatastoreSystem = hostMO.getHostDatastoreSystemMO();
  
                  String iScsiName = cmd.get_iScsiName();
  
                  ManagedObjectReference morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, VmwareResource.getDatastoreName(iScsiName));
                  DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDS);
  
                  if (path.startsWith("[-iqn.")) {
                      // Rescan 1:1 LUN that VMware may not know the LUN was recently resized
                      _storageProcessor.rescanAllHosts(context, lstHosts, true, true);
                  }
  
                  _storageProcessor.expandDatastore(hostDatastoreSystem, dsMo);
              }
  
              boolean volumePathChangeObserved = false;
              boolean datastoreChangeObserved = false;
  
              Pair<String, String> pathAndChainInfo = getNewPathAndChainInfoInDatastoreCluster(vmMo, path, chainInfo, managed, cmd.get_iScsiName(), poolUUID, cmd.getContextParam(DiskTO.PROTOCOL_TYPE));
              Pair<String, String> poolUUIDandChainInfo = getNewPoolUUIDAndChainInfoInDatastoreCluster(vmMo, path, chainInfo, managed, cmd.get_iScsiName(), poolUUID, cmd.getContextParam(DiskTO.PROTOCOL_TYPE));
  
              if (pathAndChainInfo != null) {
                  volumePathChangeObserved = true;
                  path = pathAndChainInfo.first();
                  chainInfo = pathAndChainInfo.second();
              }
  
              if (poolUUIDandChainInfo != null) {
                  datastoreChangeObserved = true;
                  poolUUID = poolUUIDandChainInfo.first();
                  chainInfo = poolUUIDandChainInfo.second();
              }
  
              // OfflineVmwareMigration: 5. ignore/replace the rest of the try-block; It is the functional bit
              VirtualDisk disk = getDiskAfterResizeDiskValidations(vmMo, path);
              String vmdkAbsFile = VmwareHelper.getAbsoluteVmdkFile(disk);
  
              if (vmdkAbsFile != null && !vmdkAbsFile.isEmpty()) {
                  vmMo.updateAdapterTypeIfRequired(vmdkAbsFile);
              }
  
              disk.setCapacityInKB(newSize);
  
              VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
  
              deviceConfigSpec.setDevice(disk);
              deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.EDIT);
  
              VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
  
              vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
  
              if (!vmMo.configureVm(vmConfigSpec)) {
                  throw new Exception(String.format("Failed to configure VM [name: %s] to resize disk.", vmName));
              }
  
              ResizeVolumeAnswer answer = new ResizeVolumeAnswer(cmd, true, "success", newSize * 1024);
              if (datastoreChangeObserved) {
                  answer.setContextParam("datastoreUUID", poolUUID);
                  answer.setContextParam("chainInfo", chainInfo);
              }
  
              if (volumePathChangeObserved) {
                  answer.setContextParam("volumePath", path);
                  answer.setContextParam("chainInfo", chainInfo);
              }
              return answer;
          } catch (Exception e) {
              String errorMsg = String.format("Failed to resize volume of VM [name: %s] due to: [%s].", vmName, e.getMessage());
              logger.error(errorMsg, e);
              return new ResizeVolumeAnswer(cmd, false, errorMsg);
          } finally {
              // OfflineVmwareMigration: 6. check if a worker was used and destroy it if needed
              try {
                  if (useWorkerVm) {
                      logger.info("Destroy worker VM after volume resize");
  
                      if (vmMo != null) {
                          vmMo.detachDisk(vmdkDataStorePath, false);
                          vmMo.destroy();
                      }
                  }
              } catch (Throwable e) {
                  logger.error(String.format("Failed to destroy worker VM [name: %s] due to: [%s].", vmName, e.getMessage()), e);
              }
          }
      }
  
      private VirtualDisk getDiskAfterResizeDiskValidations(VirtualMachineMO vmMo, String volumePath) throws Exception {
          Pair<VirtualDisk, String> vdisk = vmMo.getDiskDevice(volumePath);
          if (vdisk == null) {
              String errorMsg = String.format("Resize volume of VM [name: %s] failed because disk device [path: %s] doesn't exist.", vmMo.getVmName(), volumePath);
              logger.error(errorMsg);
              throw new Exception(errorMsg);
          }
  
          // IDE virtual disk cannot be re-sized if VM is running
          if (vdisk.second() != null && vdisk.second().toLowerCase().contains("ide")) {
              String errorMsg = String.format("Re-sizing a virtual disk over an IDE controller is not supported in the VMware hypervisor. "
                      + "Please re-try when virtual disk is attached to VM [name: %s] using a SCSI controller.", vmMo.getVmName());
              logger.error(errorMsg);
              throw new Exception(errorMsg);
          }
  
          VirtualDisk disk = vdisk.first();
          if ((VirtualDiskFlatVer2BackingInfo) disk.getBacking() != null && ((VirtualDiskFlatVer2BackingInfo) disk.getBacking()).getParent() != null) {
              String errorMsg = String.format("Resize of volume in VM [name: %s] is not supported because Disk device [path: %s] has Parents: [%s].",
                      vmMo.getVmName(), volumePath, ((VirtualDiskFlatVer2BackingInfo) disk.getBacking()).getParent().getUuid());
              logger.error(errorMsg);
              throw new Exception(errorMsg);
          }
          return disk;
      }
  
      private Pair<String, String> getNewPathAndChainInfoInDatastoreCluster(VirtualMachineMO vmMo, String path, String chainInfo, boolean managed, String iscsiName, String poolUUID, String poolType) throws Exception {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          if (poolType != null && poolType.equalsIgnoreCase(Storage.StoragePoolType.DatastoreCluster.toString())) {
              VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
              VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, path, chainInfo, managed, iscsiName, poolUUID, hyperHost, context);
              if (diskInfoBuilder != null && matchingExistingDisk != null) {
                  String[] diskChain = matchingExistingDisk.getDiskChain();
                  DatastoreFile file = new DatastoreFile(diskChain[0]);
                  if (!file.getFileBaseName().equalsIgnoreCase(path)) {
                      if (logger.isInfoEnabled())
                          logger.info("Detected disk-chain top file change on volume: " + path + " -> " + file.getFileBaseName());
                      path = file.getFileBaseName();
                      chainInfo = _gson.toJson(matchingExistingDisk);
                      return new Pair<>(path, chainInfo);
                  }
              }
          }
          return null;
      }
  
      private Pair<String, String> getNewPoolUUIDAndChainInfoInDatastoreCluster(VirtualMachineMO vmMo, String path, String chainInfo, boolean managed, String iscsiName, String poolUUID, String poolType) throws Exception {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          if (poolType != null && poolType.equalsIgnoreCase(Storage.StoragePoolType.DatastoreCluster.toString())) {
              VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
              VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, path, chainInfo, managed, iscsiName, poolUUID, hyperHost, context);
              if (diskInfoBuilder != null && matchingExistingDisk != null) {
                  String[] diskChain = matchingExistingDisk.getDiskChain();
                  DatastoreFile file = new DatastoreFile(diskChain[0]);
                  DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter());
                  DatastoreMO diskDatastoreMofromVM = new DatastoreMO(context, dcMo.findDatastore(file.getDatastoreName()));
                  if (diskDatastoreMofromVM != null) {
                      String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID);
                      if (!actualPoolUuid.equalsIgnoreCase(poolUUID)) {
                          logger.warn(String.format("Volume %s found to be in a different storage pool %s", path, actualPoolUuid));
                          poolUUID = actualPoolUuid;
                          chainInfo = _gson.toJson(matchingExistingDisk);
                          return new Pair<>(poolUUID, chainInfo);
                      }
                  }
              }
          }
          return null;
      }
  
      protected Answer execute(CheckNetworkCommand cmd) {
          // TODO setup portgroup for private network needs to be done here now
          return new CheckNetworkAnswer(cmd, true, "Network Setup check by names is done");
      }
  
      protected Answer execute(NetworkUsageCommand cmd) {
          if (cmd.isForVpc()) {
              return VPCNetworkUsage(cmd);
          }
  
          if (cmd.getOption() != null && cmd.getOption().equals("create")) {
              String result = networkUsage(cmd.getPrivateIP(), "create", null);
              return new NetworkUsageAnswer(cmd, result, 0L, 0L);
          }
          long[] stats = getNetworkStats(cmd.getPrivateIP(), null);
  
          return new NetworkUsageAnswer(cmd, "", stats[0], stats[1]);
      }
  
      protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) {
          String privateIp = cmd.getPrivateIP();
          String option = cmd.getOption();
          String publicIp = cmd.getGatewayIP();
          String vpcCIDR = cmd.getVpcCIDR();
  
          final long[] stats = getVPCNetworkStats(privateIp, publicIp, option, vpcCIDR);
          return new NetworkUsageAnswer(cmd, "", stats[0], stats[1]);
      }
  
      protected long[] getVPCNetworkStats(String privateIp, String publicIp, String option, String vpcCIDR) {
          String args = "-l " + publicIp + " ";
          if (option.equals("get")) {
              args += "-g";
          } else if (option.equals("create")) {
              args += "-c";
              args += " -v " + vpcCIDR;
          } else if (option.equals("reset")) {
              args += "-r";
          } else if (option.equals("vpn")) {
              args += "-n";
          } else if (option.equals("remove")) {
              args += "-d";
          } else {
              return new long[2];
          }
  
          ExecutionResult callResult = executeInVR(privateIp, "vpc_netusage.sh", args);
  
          if (!callResult.isSuccess()) {
              logger.error("Unable to execute NetworkUsage command on DomR (" + privateIp + "), domR may not be ready yet. failure due to " + callResult.getDetails());
          }
  
          if (option.equals("get") || option.equals("vpn")) {
              String result = callResult.getDetails();
              if (result == null || result.isEmpty()) {
                  logger.error(" vpc network usage get returns empty ");
              }
              long[] stats = new long[2];
              if (result != null) {
                  String[] splitResult = result.split(":");
                  int i = 0;
                  while (i < splitResult.length - 1) {
                      stats[0] += Long.parseLong(splitResult[i++]);
                      stats[1] += Long.parseLong(splitResult[i++]);
                  }
                  return stats;
              }
          }
          return new long[2];
      }
  
      protected long[] getNetworkLbStats(String privateIp, String publicIp, Integer port) {
          String args = publicIp + " " + port;
          ExecutionResult callResult = executeInVR(privateIp, "get_haproxy_stats.sh", args);
  
          String result = callResult.getDetails();
          if (!Boolean.TRUE.equals(callResult.isSuccess())) {
              logger.error(String.format("Unable to get network loadbalancer stats on DomR (%s), domR may not be ready yet. failure due to %s", privateIp, callResult.getDetails()));
              result = null;
          } else if (result == null || result.isEmpty()) {
              logger.error("Get network loadbalancer stats returns empty result");
          }
          long[] stats = new long[1];
          if (result != null) {
              final String[] splitResult = result.split(",");
              stats[0] += Long.parseLong(splitResult[0]);
          }
          return stats;
      }
  
      protected Answer execute(GetAutoScaleMetricsCommand cmd) {
          long bytesSent;
          long bytesReceived;
          if (cmd.isForVpc()) {
              long[] stats = getVPCNetworkStats(cmd.getPrivateIP(), cmd.getPublicIP(), "get", "");
              bytesSent = stats[0];
              bytesReceived = stats[1];
          } else {
              long [] stats = getNetworkStats(cmd.getPrivateIP(), cmd.getPublicIP());
              bytesSent = stats[0];
              bytesReceived = stats[1];
          }
  
          long [] lbStats = getNetworkLbStats(cmd.getPrivateIP(), cmd.getPublicIP(), cmd.getPort());
          long lbConnections = lbStats[0];
  
          List<VirtualRouterAutoScale.AutoScaleMetricsValue> values = new ArrayList<>();
  
          for (VirtualRouterAutoScale.AutoScaleMetrics metrics : cmd.getMetrics()) {
              switch (metrics.getCounter()) {
                  case NETWORK_RECEIVED_AVERAGE_MBPS:
                      values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED_VM_GROUP,
                              (double) bytesReceived / VirtualRouterAutoScale.MBITS_TO_BYTES));
                      break;
                  case NETWORK_TRANSMIT_AVERAGE_MBPS:
                      values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED_VM_GROUP,
                              (double) bytesSent / VirtualRouterAutoScale.MBITS_TO_BYTES));
                      break;
                  case LB_AVERAGE_CONNECTIONS:
                      values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.INSTANT_VM, (double) lbConnections));
                      break;
              }
          }
  
          return new GetAutoScaleMetricsAnswer(cmd, true, values);
      }
  
      @Override
      public ExecutionResult createFileInVR(String routerIp, String filePath, String fileName, String content) {
          File keyFile = getSystemVmKeyFile();
          try {
              SshHelper.scpTo(routerIp, 3922, "root", keyFile, null, filePath, content.getBytes(StringUtils.getPreferredCharset()), fileName, null);
          } catch (Exception e) {
              logger.warn("Fail to create file " + filePath + fileName + " in VR " + routerIp, e);
              return new ExecutionResult(false, e.getMessage());
          }
          return new ExecutionResult(true, null);
      }
  
      @Override
      public ExecutionResult prepareCommand(NetworkElementCommand cmd) {
          //Update IP used to access router
          cmd.setRouterAccessIp(getRouterSshControlIp(cmd));
          assert cmd.getRouterAccessIp() != null;
  
          if (cmd instanceof IpAssocVpcCommand) {
              return prepareNetworkElementCommand((IpAssocVpcCommand) cmd);
          } else if (cmd instanceof IpAssocCommand) {
              return prepareNetworkElementCommand((IpAssocCommand) cmd);
          } else if (cmd instanceof SetSourceNatCommand) {
              return prepareNetworkElementCommand((SetSourceNatCommand) cmd);
          } else if (cmd instanceof SetupGuestNetworkCommand) {
              return prepareNetworkElementCommand((SetupGuestNetworkCommand) cmd);
          } else if (cmd instanceof SetNetworkACLCommand) {
              return prepareNetworkElementCommand((SetNetworkACLCommand) cmd);
          }
          return new ExecutionResult(true, null);
      }
  
      @Override
      public ExecutionResult cleanupCommand(NetworkElementCommand cmd) {
          if (cmd instanceof IpAssocCommand && !(cmd instanceof IpAssocVpcCommand)) {
              return cleanupNetworkElementCommand((IpAssocCommand)cmd);
          }
          return new ExecutionResult(true, null);
      }
  
      //
      // list IP with eth devices
      //  ifconfig ethx |grep -B1 "inet addr" | awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }'
      //     | awk -F: '{ print $1 ": " $3 }'
      //
      // returns
      //      eth0:xx.xx.xx.xx
      //
      //
      private int findRouterEthDeviceIndex(String domrName, String routerIp, String mac) throws Exception {
          File keyFile = getSystemVmKeyFile();
          logger.info("findRouterEthDeviceIndex. mac: " + mac);
          ArrayList<String> skipInterfaces = new ArrayList<>(Arrays.asList("all", "default", "lo"));
  
          // when we dynamically plug in a new NIC into virtual router, it may take time to show up in guest OS
          // we use a waiting loop here as a workaround to synchronize activities in systems
          long startTick = System.currentTimeMillis();
          long waitTimeoutMillis = VmwareManager.s_vmwareNicHotplugWaitTimeout.value();
          while (System.currentTimeMillis() - startTick < waitTimeoutMillis) {
  
              // TODO : this is a temporary very inefficient solution, will refactor it later
              Pair<Boolean, String> result = SshHelper.sshExecute(routerIp, DefaultDomRSshPort, "root", keyFile, null, "ls /proc/sys/net/ipv4/conf");
              if (result.first()) {
                  String[] tokens = result.second().split("\\s+");
                  for (String token : tokens) {
                      if (!(skipInterfaces.contains(token))) {
                          String cmd = String.format("ip address show %s | grep link/ether | sed -e 's/^[ \t]*//' | cut -d' ' -f2", token);
  
                          if (logger.isDebugEnabled())
                              logger.debug("Run domr script " + cmd);
                          Pair<Boolean, String> result2 = SshHelper.sshExecute(routerIp, DefaultDomRSshPort, "root", keyFile, null,
                                  // TODO need to find the dev index inside router based on IP address
                                  cmd);
                          if (logger.isDebugEnabled())
                              logger.debug("result: " + result2.first() + ", output: " + result2.second());
  
                          if (result2.first() && result2.second().trim().equalsIgnoreCase(mac.trim())) {
                              return Integer.parseInt(token.substring(3));
                          } else {
                              skipInterfaces.add(token);
                          }
                      }
                  }
              }
  
              logger.warn("can not find intereface associated with mac: " + mac + ", guest OS may still at loading state, retry...");
  
              try {
                  Thread.currentThread();
                  Thread.sleep(1000);
              } catch (InterruptedException e) {
                  logger.debug("[ignored] interrupted while trying to get mac.");
              }
          }
  
          return -1;
      }
  
      private VirtualDevice findVirtualNicDevice(VirtualMachineMO vmMo, String mac) throws Exception {
  
          VirtualDevice[] nics = vmMo.getNicDevices();
          for (VirtualDevice nic : nics) {
              if (nic instanceof VirtualEthernetCard) {
                  if (((VirtualEthernetCard) nic).getMacAddress().equals(mac))
                      return nic;
              }
          }
          return null;
      }
  
      protected ExecutionResult prepareNetworkElementCommand(SetupGuestNetworkCommand cmd) {
          NicTO nic = cmd.getNic();
          String routerIp = getRouterSshControlIp(cmd);
          String domrName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
  
          try {
              int ethDeviceNum = findRouterEthDeviceIndex(domrName, routerIp, nic.getMac());
              nic.setDeviceId(ethDeviceNum);
          } catch (Exception e) {
              String msg = "Prepare SetupGuestNetwork failed due to " + e;
              logger.warn(msg, e);
              return new ExecutionResult(false, msg);
          }
          return new ExecutionResult(true, null);
      }
  
      private ExecutionResult prepareNetworkElementCommand(IpAssocVpcCommand cmd) {
          String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
          String routerIp = getRouterSshControlIp(cmd);
  
          try {
              IpAddressTO[] ips = cmd.getIpAddresses();
              for (IpAddressTO ip : ips) {
  
                  int ethDeviceNum = findRouterEthDeviceIndex(routerName, routerIp, ip.getVifMacAddress());
                  if (ethDeviceNum < 0) {
                      if (ip.isAdd()) {
                          throw new InternalErrorException("Failed to find DomR VIF to associate/disassociate IP with.");
                      } else {
                          logger.debug("VIF to deassociate IP with does not exist, return success");
                          continue;
                      }
                  }
  
                  ip.setNicDevId(ethDeviceNum);
              }
          } catch (Exception e) {
              logger.error("Prepare Ip Assoc failure on applying one ip due to exception:  ", e);
              return new ExecutionResult(false, e.toString());
          }
  
          return new ExecutionResult(true, null);
      }
  
      protected ExecutionResult prepareNetworkElementCommand(SetSourceNatCommand cmd) {
          String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
          String routerIp = getRouterSshControlIp(cmd);
          IpAddressTO pubIp = cmd.getIpAddress();
  
          try {
              int ethDeviceNum = findRouterEthDeviceIndex(routerName, routerIp, pubIp.getVifMacAddress());
              pubIp.setNicDevId(ethDeviceNum);
          } catch (Exception e) {
              String msg = "Prepare Ip SNAT failure due to " + e;
              logger.error(msg, e);
              return new ExecutionResult(false, e.toString());
          }
          return new ExecutionResult(true, null);
      }
  
      private ExecutionResult prepareNetworkElementCommand(SetNetworkACLCommand cmd) {
          NicTO nic = cmd.getNic();
          String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
          String routerIp = getRouterSshControlIp(cmd);
  
          try {
              int ethDeviceNum = findRouterEthDeviceIndex(routerName, routerIp, nic.getMac());
              nic.setDeviceId(ethDeviceNum);
          } catch (Exception e) {
              String msg = "Prepare SetNetworkACL failed due to " + e;
              logger.error(msg, e);
              return new ExecutionResult(false, msg);
          }
          return new ExecutionResult(true, null);
      }
  
      private PlugNicAnswer execute(PlugNicCommand cmd) {
          if (logger.isInfoEnabled()) {
              logger.info("Executing resource PlugNicCommand " + _gson.toJson(cmd));
          }
  
          try {
              VirtualEthernetCardType nicDeviceType = null;
              if (cmd.getDetails() != null) {
                  nicDeviceType = VirtualEthernetCardType.valueOf(cmd.getDetails().get("nicAdapter"));
              }
              plugNicCommandInternal(cmd.getVmName(), nicDeviceType, cmd.getNic(), cmd.getVMType());
              return new PlugNicAnswer(cmd, true, "success");
          } catch (Exception e) {
              logger.error("Unexpected exception: ", e);
              return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + e);
          }
      }
  
      private void plugNicCommandInternal(String vmName, VirtualEthernetCardType nicDeviceType, NicTO nicTo, VirtualMachine.Type vmType) throws Exception {
          getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
  
          if (vmMo == null) {
              if (hyperHost instanceof HostMO) {
                  ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
                  vmMo = clusterMo.findVmOnHyperHost(vmName);
              }
          }
  
          if (vmMo == null) {
              String msg = "Router " + vmName + " no longer exists to execute PlugNic command";
              logger.error(msg);
              throw new Exception(msg);
          }
  
              /*
              if(!isVMWareToolsInstalled(vmMo)){
                  String errMsg = "vmware tools is not installed or not running, cannot add nic to vm " + vmName;
                  logger.debug(errMsg);
                  return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + errMsg);
              }
               */
          // Fallback to E1000 if no specific nicAdapter is passed
          if (nicDeviceType == null) {
              nicDeviceType = VirtualEthernetCardType.E1000;
          }
  
          // find a usable device number in VMware environment
          VirtualDevice[] nicDevices = vmMo.getSortedNicDevices();
          int deviceNumber = -1;
          for (VirtualDevice device : nicDevices) {
              if (device.getUnitNumber() > deviceNumber)
                  deviceNumber = device.getUnitNumber();
          }
          deviceNumber++;
  
          VirtualDevice nic;
          Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, false, nicTo.getNetworkSegmentName(), vmType);
          String dvSwitchUuid;
          if (VmwareHelper.isDvPortGroup(networkInfo.first())) {
              ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
              DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
              ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
              dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
              logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
              nic = VmwareHelper.prepareDvNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), dvSwitchUuid,
                      nicTo.getMac(), deviceNumber + 1, true, true);
          } else {
              logger.info("Preparing NIC device on network " + networkInfo.second());
              nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(),
                      nicTo.getMac(), deviceNumber + 1, true, true);
          }
  
          configureNicDevice(vmMo, nic, VirtualDeviceConfigSpecOperation.ADD, "PlugNicCommand");
      }
  
      private ReplugNicAnswer execute(ReplugNicCommand cmd) {
          getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
          VmwareContext context = getServiceContext();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              String vmName = cmd.getVmName();
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  if (hyperHost instanceof HostMO) {
                      ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
                      vmMo = clusterMo.findVmOnHyperHost(vmName);
                  }
              }
  
              if (vmMo == null) {
                  String msg = "Router " + vmName + " no longer exists to execute ReplugNic command";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              /*
              if(!isVMWareToolsInstalled(vmMo)){
                  String errMsg = "vmware tools is not installed or not running, cannot add nic to vm " + vmName;
                  logger.debug(errMsg);
                  return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + errMsg);
              }
               */
              // Fallback to E1000 if no specific nicAdapter is passed
              VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.E1000;
              Map<String, String> details = cmd.getDetails();
              if (details != null) {
                  nicDeviceType = VirtualEthernetCardType.valueOf((String) details.get("nicAdapter"));
              }
  
              NicTO nicTo = cmd.getNic();
  
              VirtualDevice nic = findVirtualNicDevice(vmMo, nicTo.getMac());
              if (nic == null) {
                  return new ReplugNicAnswer(cmd, false, "Nic to replug not found");
              }
  
              Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, false, null, cmd.getVMType());
              String dvSwitchUuid;
              if (VmwareHelper.isDvPortGroup(networkInfo.first())) {
                  ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
                  DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
                  ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
                  dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
                  logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
                  VmwareHelper.updateDvNicDevice(nic, networkInfo.first(), dvSwitchUuid);
              } else {
                  logger.info("Preparing NIC device on network " + networkInfo.second());
  
                  VmwareHelper.updateNicDevice(nic, networkInfo.first(), networkInfo.second());
              }
  
              configureNicDevice(vmMo, nic, VirtualDeviceConfigSpecOperation.EDIT, "ReplugNicCommand");
  
              return new ReplugNicAnswer(cmd, true, "success");
          } catch (Exception e) {
              logger.error("Unexpected exception: ", e);
              return new ReplugNicAnswer(cmd, false, "Unable to execute ReplugNicCommand due to " + e);
          }
      }
  
      private UnPlugNicAnswer execute(UnPlugNicCommand cmd) {
          VmwareContext context = getServiceContext();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              String vmName = cmd.getVmName();
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  if (hyperHost instanceof HostMO) {
                      ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
                      vmMo = clusterMo.findVmOnHyperHost(vmName);
                  }
              }
  
              if (vmMo == null) {
                  String msg = "VM " + vmName + " no longer exists to execute UnPlugNic command";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              /*
              if(!isVMWareToolsInstalled(vmMo)){
                  String errMsg = "vmware tools not installed or not running, cannot remove nic from vm " + vmName;
                  logger.debug(errMsg);
                  return new UnPlugNicAnswer(cmd, false, "Unable to execute unPlugNicCommand due to " + errMsg);
              }
               */
              VirtualDevice nic = findVirtualNicDevice(vmMo, cmd.getNic().getMac());
              if (nic == null) {
                  return new UnPlugNicAnswer(cmd, true, "success");
              }
              configureNicDevice(vmMo, nic, VirtualDeviceConfigSpecOperation.REMOVE, "unplugNicCommand");
  
              return new UnPlugNicAnswer(cmd, true, "success");
          } catch (Exception e) {
              logger.error("Unexpected exception: ", e);
              return new UnPlugNicAnswer(cmd, false, "Unable to execute unPlugNicCommand due to " + e);
          }
      }
  
      private void plugPublicNic(VirtualMachineMO vmMo, final String vlanId, final IpAddressTO ipAddressTO) throws Exception {
          // TODO : probably need to set traffic shaping
          Pair<ManagedObjectReference, String> networkInfo;
          VirtualSwitchType vSwitchType = VirtualSwitchType.StandardVirtualSwitch;
          if (_publicTrafficInfo != null) {
              vSwitchType = _publicTrafficInfo.getVirtualSwitchType();
          }
          /** FIXME We have no clue which network this nic is on and that means that we can't figure out the BroadcastDomainType
           *  so we assume that it's VLAN for now
           */
          if (VirtualSwitchType.StandardVirtualSwitch == vSwitchType) {
              networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(),
                      "cloud.public", vmMo.getRunningHost(), vlanId, ipAddressTO.getNetworkRate(), null,
                      _opsTimeout, true, BroadcastDomainType.Vlan, null, null);
          } else {
              networkInfo =
                      HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", vmMo.getRunningHost(), vlanId, null, ipAddressTO.getNetworkRate(), null,
                              _opsTimeout, vSwitchType, _portsPerDvPortGroup, null, false, BroadcastDomainType.Vlan, _vsmCredentials, null, null);
          }
  
          int nicIndex = allocPublicNicIndex(vmMo);
  
          try {
              VirtualDevice[] nicDevices = vmMo.getSortedNicDevices();
  
              VirtualEthernetCard device = (VirtualEthernetCard) nicDevices[nicIndex];
  
              if (VirtualSwitchType.StandardVirtualSwitch == vSwitchType) {
                  VirtualEthernetCardNetworkBackingInfo nicBacking = new VirtualEthernetCardNetworkBackingInfo();
                  nicBacking.setDeviceName(networkInfo.second());
                  nicBacking.setNetwork(networkInfo.first());
                  device.setBacking(nicBacking);
              } else {
                  HostMO hostMo = vmMo.getRunningHost();
                  DatacenterMO dataCenterMo = new DatacenterMO(hostMo.getContext(), hostMo.getHyperHostDatacenter());
                  device.setBacking(dataCenterMo.getDvPortBackingInfo(networkInfo));
              }
  
              configureNicDevice(vmMo, device, VirtualDeviceConfigSpecOperation.EDIT, "plugPublicNic");
          } catch (Exception e) {
  
              // restore allocation mask in case of exceptions
              String nicMasksStr = vmMo.getCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK);
              int nicMasks = Integer.parseInt(nicMasksStr);
              nicMasks &= ~(1 << nicIndex);
              vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMasks));
  
              throw e;
          }
      }
  
      private int allocPublicNicIndex(VirtualMachineMO vmMo) throws Exception {
          String nicMasksStr = vmMo.getCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK);
          if (nicMasksStr == null || nicMasksStr.isEmpty()) {
              throw new Exception("Could not find NIC allocation info");
          }
  
          int nicMasks = Integer.parseInt(nicMasksStr);
          VirtualDevice[] nicDevices = vmMo.getNicDevices();
          for (int i = 3; i < nicDevices.length; i++) {
              if ((nicMasks & (1 << i)) == 0) {
                  nicMasks |= (1 << i);
                  vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMasks));
                  return i;
              }
          }
  
          throw new Exception("Could not allocate a free public NIC");
      }
  
      private ExecutionResult prepareNetworkElementCommand(IpAssocCommand cmd) {
          VmwareContext context = getServiceContext();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              IpAddressTO[] ips = cmd.getIpAddresses();
              String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
              String controlIp = getRouterSshControlIp(cmd);
  
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(routerName);
  
              // command may sometimes be redirect to a wrong host, we relax
              // the check and will try to find it within cluster
              if (vmMo == null) {
                  if (hyperHost instanceof HostMO) {
                      ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
                      vmMo = clusterMo.findVmOnHyperHost(routerName);
                  }
              }
  
              if (vmMo == null) {
                  String msg = "Router " + routerName + " no longer exists to execute IPAssoc command";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              for (IpAddressTO ip : ips) {
                  /**
                   * TODO support other networks
                   */
                  URI broadcastUri = BroadcastDomainType.fromString(ip.getBroadcastUri());
                  if (BroadcastDomainType.getSchemeValue(broadcastUri) != BroadcastDomainType.Vlan) {
                      throw new InternalErrorException("Unable to assign a public IP to a VIF on network " + ip.getBroadcastUri());
                  }
                  String vlanId = BroadcastDomainType.getValue(broadcastUri);
  
                  String publicNetworkName = HypervisorHostHelper.getPublicNetworkNamePrefix(vlanId);
                  Pair<Integer, VirtualDevice> publicNicInfo = vmMo.getNicDeviceIndex(publicNetworkName);
  
                  if (logger.isDebugEnabled()) {
                      logger.debug("Find public NIC index, public network name: " + publicNetworkName + ", index: " + publicNicInfo.first());
                  }
  
                  boolean addVif = false;
                  if (ip.isAdd() && publicNicInfo.first() == -1) {
                      if (logger.isDebugEnabled()) {
                          logger.debug("Plug new NIC to associate" + controlIp + " to " + ip.getPublicIp());
                      }
                      addVif = true;
                  }
  
                  if (addVif) {
                      NicTO nicTO = ip.getNicTO();
                      VirtualEthernetCardType nicDeviceType = null;
                      if (ip.getDetails() != null) {
                          nicDeviceType = VirtualEthernetCardType.valueOf(ip.getDetails().get("nicAdapter"));
                      }
                      plugNicCommandInternal(routerName, nicDeviceType, nicTO, VirtualMachine.Type.DomainRouter);
                      publicNicInfo = vmMo.getNicDeviceIndex(publicNetworkName);
                      if (publicNicInfo.first() >= 0) {
                          networkUsage(controlIp, "addVif", "eth" + publicNicInfo.first());
                      }
                  }
  
                  if (publicNicInfo.first() < 0) {
                      String msg = "Failed to find DomR VIF to associate/disassociate IP with.";
                      logger.error(msg);
                      throw new InternalErrorException(msg);
                  }
                  ip.setNicDevId(publicNicInfo.first());
                  ip.setNewNic(addVif);
              }
          } catch (Throwable e) {
              logger.error("Unexpected exception: " + e + " will shortcut rest of IPAssoc commands", e);
              return new ExecutionResult(false, e.toString());
          }
          return new ExecutionResult(true, null);
      }
  
      private ExecutionResult cleanupNetworkElementCommand(IpAssocCommand cmd) {
          VmwareContext context = getServiceContext();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              IpAddressTO[] ips = cmd.getIpAddresses();
              String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
  
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(routerName);
              // command may sometimes be redirected to a wrong host, we relax
              // the check and will try to find it within datacenter
              if (vmMo == null) {
                  if (hyperHost instanceof HostMO) {
                      final DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter());
                      vmMo = dcMo.findVm(routerName);
                  }
              }
  
              if (vmMo == null) {
                  String msg = String.format("Router %s no longer exists to execute IPAssoc command ", routerName);
                  logger.error(msg);
                  throw new Exception(msg);
              }
              final String lastIp = cmd.getAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP);
              for (IpAddressTO ip : ips) {
                  if (ip.isAdd() || lastIp.equalsIgnoreCase("false")) {
                      continue;
                  }
                  Pair<VirtualDevice, Integer> nicInfo = getVirtualDevice(vmMo, ip);
  
                  if (nicInfo.second() == 2) {
                      return new ExecutionResult(true, "Not removing eth2 in network VR because it is the public NIC of source NAT");
                  }
                  if (nicInfo.first() == null) {
                      return new ExecutionResult(false, "Couldn't find NIC");
                  }
                  configureNicDevice(vmMo, nicInfo.first(), VirtualDeviceConfigSpecOperation.REMOVE, "unplugNicCommand");
              }
          } catch (Throwable e) {
              logger.error("Unexpected exception: " + e + " will shortcut rest of IPAssoc commands", e);
              return new ExecutionResult(false, e.toString());
          }
          return new ExecutionResult(true, null);
      }
  
      private Pair<VirtualDevice, Integer> getVirtualDevice(VirtualMachineMO vmMo, IpAddressTO ip) throws Exception {
          NicTO nicTO = ip.getNicTO();
          URI broadcastUri = BroadcastDomainType.fromString(ip.getBroadcastUri());
          if (BroadcastDomainType.getSchemeValue(broadcastUri) != BroadcastDomainType.Vlan) {
              throw new InternalErrorException(String.format("Unable to assign a public IP to a VIF on network %s", ip.getBroadcastUri()));
          }
          String vlanId = BroadcastDomainType.getValue(broadcastUri);
  
          String publicNetworkName = HypervisorHostHelper.getPublicNetworkNamePrefix(vlanId);
          Pair<Integer, VirtualDevice> publicNicInfo = vmMo.getNicDeviceIndex(publicNetworkName);
  
          if (logger.isDebugEnabled()) {
              logger.debug(String.format("Find public NIC index, public network name: %s , index: %s", publicNetworkName, publicNicInfo.first()));
          }
  
          return new Pair<>(findVirtualNicDevice(vmMo, nicTO.getMac()), publicNicInfo.first());
      }
  
      private void configureNicDevice(VirtualMachineMO vmMo, VirtualDevice nic, VirtualDeviceConfigSpecOperation operation, String commandName) throws Exception {
          VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
          VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
          deviceConfigSpec.setDevice(nic);
          deviceConfigSpec.setOperation(operation);
  
  
          vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
          if (!vmMo.configureVm(vmConfigSpec)) {
              throw new Exception(String.format("Failed to configure devices when running %s", commandName));
          }
      }
  
      @Override
      public ExecutionResult executeInVR(String routerIP, String script, String args) {
          return executeInVR(routerIP, script, args, VRScripts.VR_SCRIPT_EXEC_TIMEOUT);
      }
  
      @Override
      public ExecutionResult executeInVR(String routerIP, String script, String args, Duration timeout) {
          Pair<Boolean, String> result;
  
          //TODO: Password should be masked, cannot output to log directly
          if (logger.isDebugEnabled()) {
              logger.debug("Run command on VR: " + routerIP + ", script: " + script + " with args: " + args);
          }
  
          try {
              result = SshHelper.sshExecute(routerIP, DefaultDomRSshPort, "root", getSystemVmKeyFile(), null, "/opt/cloud/bin/" + script + " " + args,
                      VRScripts.CONNECTION_TIMEOUT, VRScripts.CONNECTION_TIMEOUT, timeout);
          } catch (Exception e) {
              String msg = "Command failed due to " + VmwareHelper.getExceptionMessage(e);
              logger.error(msg);
              result = new Pair<>(false, msg);
          }
          if (logger.isDebugEnabled()) {
              logger.debug(script + " execution result: " + result.first().toString());
          }
          return new ExecutionResult(result.first(), result.second());
      }
  
      protected CheckSshAnswer execute(CheckSshCommand cmd) {
          String vmName = cmd.getName();
          String privateIp = cmd.getIp();
          int cmdPort = cmd.getPort();
  
          if (logger.isDebugEnabled()) {
              logger.debug("Ping command port, " + privateIp + ":" + cmdPort);
          }
  
          String errorMessage = "Can not ping System VM [%s], due to: [%s].";
          try {
              String result = connect(cmd.getName(), privateIp, cmdPort);
              if (result != null) {
                  logger.error(String.format(errorMessage, vmName, result));
                  return new CheckSshAnswer(cmd, String.format(errorMessage, vmName, result));
              }
          } catch (Exception e) {
              logger.error(String.format(errorMessage, vmName, e.getMessage()), e);
              return new CheckSshAnswer(cmd, e);
          }
  
          if (logger.isDebugEnabled()) {
              logger.debug("Ping command port succeeded for vm " + vmName);
          }
  
          if (VirtualMachineName.isValidRouterName(vmName)) {
              if (logger.isDebugEnabled()) {
                  logger.debug("Execute network usage setup command on " + vmName);
              }
              networkUsage(privateIp, "create", null);
          }
  
          return new CheckSshAnswer(cmd);
      }
  
      private DiskTO[] validateDisks(DiskTO[] disks) {
          List<DiskTO> validatedDisks = new ArrayList<>();
  
          for (DiskTO vol : disks) {
              if (vol.getType() != Volume.Type.ISO) {
                  VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
                  DataStoreTO primaryStore = volumeTO.getDataStore();
                  if (primaryStore.getUuid() != null && !primaryStore.getUuid().isEmpty()) {
                      validatedDisks.add(vol);
                  }
              } else if (vol.getType() == Volume.Type.ISO) {
                  TemplateObjectTO templateTO = (TemplateObjectTO) vol.getData();
                  if (templateTO.getPath() != null && !templateTO.getPath().isEmpty()) {
                      validatedDisks.add(vol);
                  }
              } else {
                  if (logger.isDebugEnabled()) {
                      logger.debug("Drop invalid disk option, volumeTO: " + _gson.toJson(vol));
                  }
              }
          }
          Collections.sort(validatedDisks, (d1, d2) -> d1.getDiskSeq().compareTo(d2.getDiskSeq()));
          return validatedDisks.toArray(new DiskTO[0]);
      }
  
      private static DiskTO getIsoDiskTO(DiskTO[] disks) {
          for (DiskTO vol : disks) {
              if (vol.getType() == Volume.Type.ISO) {
                  return vol;
              }
          }
          return null;
      }
  
      protected ScaleVmAnswer execute(ScaleVmCommand cmd) {
          VmwareContext context = getServiceContext();
          VirtualMachineTO vmSpec = cmd.getVirtualMachine();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
              int ramMb = getReservedMemoryMb(vmSpec);
              long hotaddIncrementSizeInMb;
              long hotaddMemoryLimitInMb;
              long requestedMaxMemoryInMb = vmSpec.getMaxRam() / (1024 * 1024);
  
              // Check if VM is really running on hypervisor host
              if (getVmPowerState(vmMo) != PowerState.PowerOn) {
                  throw new CloudRuntimeException("Found that the VM " + vmMo.getVmName() + " is not running. Unable to scale-up this VM");
              }
  
              // Check max hot add limit
              hotaddIncrementSizeInMb = vmMo.getHotAddMemoryIncrementSizeInMb();
              hotaddMemoryLimitInMb = vmMo.getHotAddMemoryLimitInMb();
              if (requestedMaxMemoryInMb > hotaddMemoryLimitInMb) {
                  throw new CloudRuntimeException("Memory of VM " + vmMo.getVmName() + " cannot be scaled to " + requestedMaxMemoryInMb + "MB."
                          + " Requested memory limit is beyond the hotadd memory limit for this VM at the moment is " + hotaddMemoryLimitInMb + "MB.");
              }
  
              // Check increment is multiple of increment size
              long reminder = hotaddIncrementSizeInMb > 0 ? requestedMaxMemoryInMb % hotaddIncrementSizeInMb : 0;
              if (reminder != 0) {
                  requestedMaxMemoryInMb = requestedMaxMemoryInMb + hotaddIncrementSizeInMb - reminder;
              }
  
              // Check if license supports the feature
              VmwareHelper.isFeatureLicensed(hyperHost, FeatureKeyConstants.HOTPLUG);
              VmwareHelper.setVmScaleUpConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) requestedMaxMemoryInMb, ramMb,
                      vmSpec.isLimitCpuUse());
  
              if (!vmMo.configureVm(vmConfigSpec)) {
                  throw new Exception("Unable to execute ScaleVmCommand");
              }
          } catch (Exception e) {
              logger.error(String.format("ScaleVmCommand failed due to: [%s].", VmwareHelper.getExceptionMessage(e)), e);
              return new ScaleVmAnswer(cmd, false, String.format("Unable to execute ScaleVmCommand due to: [%s].", e));
          }
          return new ScaleVmAnswer(cmd, true, null);
      }
  
      protected void ensureDiskControllers(VirtualMachineMO vmMo, Pair<String, String> controllerInfo) throws Exception {
          if (vmMo == null) {
              return;
          }
  
          Pair<String, String> chosenDiskControllers = VmwareHelper.chooseRequiredDiskControllers(controllerInfo, vmMo, null, null);
          String scsiDiskController = HypervisorHostHelper.getScsiController(chosenDiskControllers);
          if (scsiDiskController == null) {
              return;
          }
  
          vmMo.getScsiDeviceControllerKeyNoException();
          // This VM needs SCSI controllers.
          // Get count of existing scsi controllers. Helps not to attempt to create more than the maximum allowed 4
          // Get maximum among the bus numbers in use by scsi controllers. Safe to pick maximum, because we always go sequential allocating bus numbers.
          Ternary<Integer, Integer, DiskControllerType> scsiControllerInfo = vmMo.getScsiControllerInfo();
          int requiredNumScsiControllers = VmwareHelper.MAX_SCSI_CONTROLLER_COUNT - scsiControllerInfo.first();
          int availableBusNum = scsiControllerInfo.second() + 1; // method returned current max. bus number
  
          if (DiskControllerType.getType(scsiDiskController) != scsiControllerInfo.third()) {
              logger.debug(String.format("Change controller type from: %s to: %s", scsiControllerInfo.third().toString(),
                      scsiDiskController));
              vmMo.tearDownDevices(new Class<?>[]{VirtualSCSIController.class});
              vmMo.addScsiDeviceControllers(DiskControllerType.getType(scsiDiskController));
              return;
          }
  
          if (requiredNumScsiControllers == 0) {
              return;
          }
          if (scsiControllerInfo.first() > 0) {
              // For VMs which already have a SCSI controller, do NOT attempt to add any more SCSI controllers & return the sub type.
              // For Legacy VMs would have only 1 LsiLogic Parallel SCSI controller, and doesn't require more.
              // For VMs created post device ordering support, 4 SCSI subtype controllers are ensured during deployment itself. No need to add more.
              // For fresh VM deployment only, all required controllers should be ensured.
              return;
          }
          ensureScsiDiskControllers(vmMo, scsiDiskController, requiredNumScsiControllers, availableBusNum);
      }
  
      private void ensureScsiDiskControllers(VirtualMachineMO vmMo, String scsiDiskController, int requiredNumScsiControllers, int availableBusNum) throws Exception {
          // Pick the sub type of scsi
          if (DiskControllerType.getType(scsiDiskController) == DiskControllerType.pvscsi) {
              if (!vmMo.isPvScsiSupported()) {
                  String msg = "This VM doesn't support Vmware Paravirtual SCSI controller for virtual disks, because the virtual hardware version is less than 7.";
                  throw new Exception(msg);
              }
              vmMo.ensurePvScsiDeviceController(requiredNumScsiControllers, availableBusNum);
          } else if (DiskControllerType.getType(scsiDiskController) == DiskControllerType.lsisas1068) {
              vmMo.ensureLsiLogicSasDeviceControllers(requiredNumScsiControllers, availableBusNum);
          } else if (DiskControllerType.getType(scsiDiskController) == DiskControllerType.buslogic) {
              vmMo.ensureBusLogicDeviceControllers(requiredNumScsiControllers, availableBusNum);
          } else if (DiskControllerType.getType(scsiDiskController) == DiskControllerType.lsilogic) {
              vmMo.ensureLsiLogicDeviceControllers(requiredNumScsiControllers, availableBusNum);
          }
      }
  
      protected StartAnswer execute(StartCommand cmd) {
          VirtualMachineTO vmSpec = cmd.getVirtualMachine();
          boolean vmAlreadyExistsInVcenter = false;
  
          String existingVmName = null;
          VirtualMachineFileInfo existingVmFileInfo = null;
          VirtualMachineFileLayoutEx existingVmFileLayout = null;
          List<DatastoreMO> existingDatastores = new ArrayList<>();
          String diskStoragePolicyId;
          String vmStoragePolicyId = null;
          VirtualMachineDefinedProfileSpec diskProfileSpec = null;
          VirtualMachineDefinedProfileSpec vmProfileSpec = null;
  
          DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo();
          boolean deployAsIs = deployAsIsInfo != null;
  
          Pair<String, String> names = composeVmNames(vmSpec);
          String vmInternalCSName = names.first();
          String vmNameOnVcenter = names.second();
          DiskTO rootDiskTO = null;
          String bootMode = getBootModeFromVmSpec(vmSpec, deployAsIs);
          Pair<String, String> controllerInfo = getControllerInfoFromVmSpec(vmSpec);
  
          Boolean systemVm = vmSpec.getType().isUsedBySystem();
          // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name.
          VmwareContext context = getServiceContext();
          DatacenterMO dcMo = null;
          try {
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter());
  
              // Validate VM name is unique in Datacenter
              VirtualMachineMO vmInVcenter = dcMo.checkIfVmAlreadyExistsInVcenter(vmNameOnVcenter, vmInternalCSName);
              if (vmInVcenter != null) {
                  vmAlreadyExistsInVcenter = true;
                  String msg = "VM with name: " + vmNameOnVcenter + " already exists in vCenter.";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              DiskTO[] specDisks = vmSpec.getDisks();
              String guestOsId = getGuestOsIdFromVmSpec(vmSpec, deployAsIs);
              DiskTO[] disks = validateDisks(vmSpec.getDisks());
              assert (disks.length > 0);
              NicTO[] nics = vmSpec.getNics();
  
              HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> dataStoresDetails = inferDatastoreDetailsFromDiskInfo(hyperHost, context, disks, cmd);
              if ((dataStoresDetails == null) || (dataStoresDetails.isEmpty())) {
                  String msg = "Unable to locate datastore details of the volumes to be attached";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              VirtualMachineDiskInfoBuilder diskInfoBuilder = null;
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
              DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic;
              int firstScsiControllerBusNum = 0;
              int numScsiControllerForSystemVm = 1;
              boolean hasSnapshot = false;
  
              List<Pair<Integer, ManagedObjectReference>> diskDatastores = null;
              if (vmMo != null) {
                  logger.info("VM " + vmInternalCSName + " already exists, tear down devices for reconfiguration");
                  if (getVmPowerState(vmMo) != PowerState.PowerOff)
                      vmMo.safePowerOff(_shutdownWaitMs);
  
                  // retrieve disk information before we tear down
                  diskDatastores = vmMo.getAllDiskDatastores();
                  diskInfoBuilder = vmMo.getDiskInfoBuilder();
                  hasSnapshot = vmMo.hasSnapshot();
  
                  tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
                  ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
                          numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
              } else {
                  ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter();
                  assert (morDc != null);
  
                  vmMo = hyperHost.findVmOnPeerHyperHost(vmInternalCSName);
                  if (vmMo != null) {
                      if (logger.isInfoEnabled()) {
                          logger.info("Found vm " + vmInternalCSName + " at other host, relocate to " + hyperHost.getHyperHostName());
                      }
  
                      takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
                      vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
  
                      if (vmMo != null) {
                          if (getVmPowerState(vmMo) != PowerState.PowerOff)
                              vmMo.safePowerOff(_shutdownWaitMs);
  
                          diskInfoBuilder = vmMo.getDiskInfoBuilder();
                          hasSnapshot = vmMo.hasSnapshot();
                          diskDatastores = vmMo.getAllDiskDatastores();
  
                          tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
                          ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
                                  numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
                      }
                  } else {
                      // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration).
                      VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName);
                      if (existingVmInDc != null) {
                          logger.debug("Found VM: " + vmInternalCSName + " on a host in a different cluster. Unregistering the exisitng VM.");
                          existingVmName = existingVmInDc.getName();
                          existingVmFileInfo = existingVmInDc.getFileInfo();
                          existingVmFileLayout = existingVmInDc.getFileLayout();
                          existingDatastores = existingVmInDc.getAllDatastores();
                          existingVmInDc.unregisterVm();
                      }
  
                      if (deployAsIs) {
                          vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
                          if (vmMo == null) {
                              logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it");
                              takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
                          }
                      } else {
                          DiskTO rootDisk = null;
                          for (DiskTO vol : disks) {
                              if (vol.getType() == Volume.Type.ROOT) {
                                  rootDisk = vol;
                              }
                          }
                          Pair<ManagedObjectReference, DatastoreMO> rootDiskDataStoreDetails = getDatastoreThatDiskIsOn(dataStoresDetails, rootDisk);
                          assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null);
                          DatastoreMO dsRootVolumeIsOn = rootDiskDataStoreDetails.second();
                          if (dsRootVolumeIsOn == null) {
                                  String msg = "Unable to locate datastore details of root volume";
                                  logger.error(msg);
                                  throw new Exception(msg);
                              }
                          if (rootDisk.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && rootDisk.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase(Storage.StoragePoolType.DatastoreCluster.toString())) {
                              if (diskInfoBuilder != null) {
                                  DatastoreMO diskDatastoreMofromVM = getDataStoreWhereDiskExists(hyperHost, context, diskInfoBuilder, rootDisk, diskDatastores);
                                  if (diskDatastoreMofromVM != null) {
                                      String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID);
                                      if (!actualPoolUuid.equalsIgnoreCase(rootDisk.getData().getDataStore().getUuid())) {
                                          dsRootVolumeIsOn = diskDatastoreMofromVM;
                                      }
                                  }
                              }
                          }
  
                          boolean vmFolderExists = dsRootVolumeIsOn.folderExists(String.format("[%s]", dsRootVolumeIsOn.getName()), vmNameOnVcenter);
                          String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value());
                          if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present.
                              registerVm(vmNameOnVcenter, dsRootVolumeIsOn);
                              vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
                              if (vmMo != null) {
                                  if (logger.isDebugEnabled()) {
                                      logger.debug("Found registered vm " + vmInternalCSName + " at host " + hyperHost.getHyperHostName());
                                  }
                              }
                              tearDownVm(vmMo);
                          } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec),
                                  vmSpec.isLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false,
                                  controllerInfo, systemVm)) {
                              throw new Exception("Failed to create VM. vmName: " + vmInternalCSName);
                          }
                      }
                  }
  
                  vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
                  if (vmMo == null) {
                      throw new Exception("Failed to find the newly create or relocated VM. vmName: " + vmInternalCSName);
                  }
              }
              if (deployAsIs && !vmMo.hasSnapshot()) {
                  logger.info("Mapping VM disks to spec disks and tearing down datadisks (if any)");
                  mapSpecDisksToClonedDisksAndTearDownDatadisks(vmMo, vmInternalCSName, specDisks);
              }
  
              int disksChanges = getDisksChangesNumberFromDisksSpec(disks, deployAsIs);
              int totalChangeDevices = disksChanges + nics.length;
              if (deployAsIsInfo != null && deployAsIsInfo.getProperties() != null) {
                  totalChangeDevices++;
              }
  
              DiskTO volIso = null;
              if (vmSpec.getType() != VirtualMachine.Type.User) {
                  // system VM needs a patch ISO
                  totalChangeDevices++;
              } else {
                  volIso = getIsoDiskTO(disks);
                  if (volIso == null && !deployAsIs) {
                      totalChangeDevices++;
                  }
              }
  
              VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
  
              int i = 0;
              int ideUnitNumber = !deployAsIs ? 0 : vmMo.getNextIDEDeviceNumber();
              int scsiUnitNumber = !deployAsIs ? 0 : vmMo.getNextScsiDiskDeviceNumber();
              int ideControllerKey = vmMo.getIDEDeviceControllerKey();
              int scsiControllerKey = vmMo.getScsiDeviceControllerKeyNoException();
              VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices];
              DiskTO[] sortedDisks = sortVolumesByDeviceId(disks);
              VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)),
                      getReservedMemoryMb(vmSpec), guestOsId, vmSpec.isLimitCpuUse(), deployAsIs);
  
              // Check for multi-cores per socket settings
              int numCoresPerSocket = 1;
              String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET);
              if (coresPerSocket != null) {
                  String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext());
                  // Property 'numCoresPerSocket' is supported since vSphere API 5.0
                  if (apiVersion.compareTo("5.0") >= 0) {
                      numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1);
                      vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket);
                  }
              }
  
              // Check for hotadd settings
              vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm());
              String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion();
              if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) {
                  logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be"
                          + " enabled for Virtual Machine: " + vmInternalCSName);
                  vmConfigSpec.setCpuHotAddEnabled(false);
              } else {
                  vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm());
              }
  
              if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                  logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
              }
  
              if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                  logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
              }
  
              configNestedHVSupport(vmMo, vmSpec, vmConfigSpec);
  
              //
              // Setup ISO device
              //
  
              // prepare systemvm patch ISO
              if (vmSpec.getType() != VirtualMachine.Type.User) {
                  // attach ISO (for patching of system VM)
                  Pair<String, Long> secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId));
                  String secStoreUrl = secStoreUrlAndId.first();
                  if (secStoreUrl == null) {
                      String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used",
                              _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100));
                      throw new Exception(msg);
                  }
  
                  ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl);
                  if (morSecDs == null) {
                      String msg = "Failed to prepare secondary storage on host, NFS secondary or cache store url: " + secStoreUrl + " in dc "+ _dcId;
                      throw new Exception(msg);
                  }
                  DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs);
  
                  deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
                  Pair<VirtualDevice, Boolean> isoInfo = VmwareHelper.prepareIsoDevice(vmMo,
                          null, secDsMo.getMor(), true, true, ideUnitNumber++, i + 1);
                  deviceConfigSpecArray[i].setDevice(isoInfo.first());
                  if (isoInfo.second()) {
                      if (logger.isDebugEnabled())
                          logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first()));
                      deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
                  } else {
                      if (logger.isDebugEnabled())
                          logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
                      deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT);
                  }
                  i++;
              } else if (!deployAsIs) {
                  // Note: we will always plug a CDROM device
                  if (volIso != null) {
                      for (DiskTO vol : disks) {
                          if (vol.getType() == Volume.Type.ISO) {
                              configureIso(hyperHost, vmMo, vol, deviceConfigSpecArray, ideUnitNumber++, i);
                              i++;
                          }
                      }
                  } else {
                      deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
                      Pair<VirtualDevice, Boolean> isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1);
                      deviceConfigSpecArray[i].setDevice(isoInfo.first());
                      if (isoInfo.second()) {
                          if (logger.isDebugEnabled())
                              logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
  
                          deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
                      } else {
                          if (logger.isDebugEnabled())
                              logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
  
                          deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT);
                      }
                      i++;
                  }
              }
  
              int controllerKey;
              Pair<String, String> chosenDiskControllers = VmwareHelper.chooseRequiredDiskControllers(controllerInfo,vmMo, null, null);
  
              //
              // Setup ROOT/DATA disk devices
              //
              if (multipleIsosAtached(sortedDisks) && deployAsIs) {
                  sortedDisks = getDisks(sortedDisks);
              }
  
              for (DiskTO vol : sortedDisks) {
                  if (vol.getType() == Volume.Type.ISO) {
                      if (deployAsIs) {
                          configureIso(hyperHost, vmMo, vol, deviceConfigSpecArray, ideUnitNumber++, i);
                          i++;
                      }
                      continue;
                  }
  
                  VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context);
                  Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails = getVolumeDatastoreDetails(vol, dataStoresDetails);
                  syncVolumeDatastoreAndPathForDatastoreCluster(vol, diskInfoBuilder, matchingExistingDisk, volumeDsDetails, diskDatastores, hyperHost, context);
  
                  if (deployAsIs && vol.getType() == Volume.Type.ROOT) {
                      rootDiskTO = vol;
                      resizeRootDiskOnVMStart(vmMo, rootDiskTO, hyperHost, context);
                      continue;
                  }
  
                  String diskController = getDiskController(vmMo, matchingExistingDisk, vol, chosenDiskControllers, deployAsIs);
                  if (DiskControllerType.getType(diskController) == DiskControllerType.ide) {
                      controllerKey = vmMo.getIDEControllerKey(ideUnitNumber);
                      if (vol.getType() == Volume.Type.DATADISK) {
                          // Could be result of flip due to user configured setting or "osdefault" for data disks
                          // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume
                          if (vmMo.getNumberOfVirtualDisks() > 3) {
                              throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over "
                                      + diskController + " controller, as maximum number of devices supported over IDE controller is 4 including CDROM device.");
                          }
                      }
                  } else {
                      if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) {
                          scsiUnitNumber++;
                      }
  
                      controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber);
                      if (controllerKey == -1) {
                          // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault"
                          // Retrieve existing controller and use.
                          Ternary<Integer, Integer, DiskControllerType> vmScsiControllerInfo = vmMo.getScsiControllerInfo();
                          DiskControllerType existingControllerType = vmScsiControllerInfo.third();
                          controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber);
                      }
                  }
                  if (!hasSnapshot) {
                      deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
  
                      String[] diskChain = syncDiskChain(dcMo, vmMo, vol, matchingExistingDisk, volumeDsDetails.second());
  
                      int deviceNumber;
                      if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) {
                          deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER;
                          ideUnitNumber++;
                      } else {
                          deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER;
                          scsiUnitNumber++;
                      }
  
                      VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
                      Long maxIops = volumeTO.getIopsWriteRate() + volumeTO.getIopsReadRate();
                      VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1, maxIops);
                      logger.debug(LogUtils.logGsonWithoutException("The following definitions will be used to start the VM: virtual device [%s], volume [%s].", device, volumeTO));
  
                      diskStoragePolicyId = volumeTO.getvSphereStoragePolicyId();
                      if (StringUtils.isNotEmpty(diskStoragePolicyId)) {
                          PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(context);
                          diskProfileSpec = profMgrMo.getProfileSpec(diskStoragePolicyId);
                          deviceConfigSpecArray[i].getProfile().add(diskProfileSpec);
                          if (logger.isDebugEnabled()) {
                              logger.debug(String.format("Adding vSphere storage profile: %s to virtual disk [%s]", diskStoragePolicyId, _gson.toJson(device)));
                          }
                      }
                      if (vol.getType() == Volume.Type.ROOT) {
                          rootDiskTO = vol;
                          vmStoragePolicyId = diskStoragePolicyId;
                          vmProfileSpec = diskProfileSpec;
                      }
                      deviceConfigSpecArray[i].setDevice(device);
                      deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
  
                      if (logger.isDebugEnabled())
                          logger.debug("Prepare volume at new device " + _gson.toJson(device));
  
                      i++;
                  } else {
                      if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber))
                          ideUnitNumber++;
                      else
                          scsiUnitNumber++;
                  }
              }
  
              //
              // Setup USB devices
              //
              if (StringUtils.isNotBlank(guestOsId) && guestOsId.startsWith("darwin")) { //Mac OS
                  VirtualDevice[] devices = vmMo.getMatchedDevices(new Class<?>[]{VirtualUSBController.class});
                  if (devices.length == 0) {
                      logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName);
  
                      //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access.
                      VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice();
                      deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
                      deviceConfigSpecArray[i].setDevice(usbControllerDevice);
                      deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
  
                      if (logger.isDebugEnabled())
                          logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i]));
  
                      i++;
                  } else {
                      logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName);
                  }
              }
  
              //
              // Setup NIC devices
              //
              VirtualDevice nic;
              int nicMask = 0;
              int nicCount = 0;
  
              VirtualEthernetCardType nicDeviceType;
  
              NiciraNvpApiVersion.logNiciraApiVersion();
  
              Map<String, String> nicUuidToDvSwitchUuid = new HashMap<>();
              for (NicTO nicTo : sortNicsByDeviceId(nics)) {
                  logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo));
  
                  String adapterTypeStr = deployAsIs ?
                          mapAdapterType(deployAsIsInfo.getNicAdapterMap().get(nicTo.getDeviceId())) :
                          vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER);
                  nicDeviceType = VirtualEthernetCardType.valueOf(adapterTypeStr);
  
                  if (logger.isDebugEnabled()) {
                      logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType + " on NIC device " + nicTo.getDeviceId());
                  }
                  boolean configureVServiceInNexus = (nicTo.getType() == TrafficType.Guest) && (vmSpec.getDetails().containsKey("ConfigureVServiceInNexus"));
                  VirtualMachine.Type vmType = cmd.getVirtualMachine().getType();
                  Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, configureVServiceInNexus,
                          vmSpec.getNetworkIdToNetworkNameMap().getOrDefault(nicTo.getNetworkId(), null), vmType);
                  if ((nicTo.getBroadcastType() != BroadcastDomainType.Lswitch)
                          || (nicTo.getBroadcastType() == BroadcastDomainType.Lswitch && NiciraNvpApiVersion.isApiVersionLowerThan("4.2"))) {
                      if (VmwareHelper.isDvPortGroup(networkInfo.first())) {
                          String dvSwitchUuid;
                          ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
                          DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
                          ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
                          dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
                          logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
                          nic = VmwareHelper.prepareDvNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), dvSwitchUuid,
                                  nicTo.getMac(), i + 1, true, true);
                          if (nicTo.getUuid() != null) {
                              nicUuidToDvSwitchUuid.put(nicTo.getUuid(), dvSwitchUuid);
                          }
                      } else {
                          logger.info("Preparing NIC device on network " + networkInfo.second());
                          nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(),
                                  nicTo.getMac(), i + 1, true, true);
                      }
                  } else {
                      //if NSX API VERSION >= 4.2, connect to br-int (nsx.network), do not create portgroup else previous behaviour
                      nic = VmwareHelper.prepareNicOpaque(vmMo, nicDeviceType, networkInfo.second(),
                              nicTo.getMac(), i + 1, true, true);
                  }
  
                  deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
                  deviceConfigSpecArray[i].setDevice(nic);
                  deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
  
                  if (logger.isDebugEnabled())
                      logger.debug("Prepare NIC at new device " + _gson.toJson(deviceConfigSpecArray[i]));
  
                  // this is really a hacking for DomR, upon DomR startup, we will reset all the NIC allocation after eth3
                  if (nicCount < 3)
                      nicMask |= (1 << nicCount);
  
                  i++;
                  nicCount++;
              }
  
              for (int j = 0; j < i; j++)
                  vmConfigSpec.getDeviceChange().add(deviceConfigSpecArray[j]);
  
              //
              // Setup VM options
              //
  
              // pass boot arguments through machine.id & perform customized options to VMX
              ArrayList<OptionValue> extraOptions = new ArrayList<>();
              configBasicExtraOption(extraOptions, vmSpec);
  
              if (deployAsIs) {
                  setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec, hyperHost);
              }
  
              configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid);
              configCustomExtraOption(extraOptions, vmSpec);
  
              // config for NCC
              VirtualMachine.Type vmType = cmd.getVirtualMachine().getType();
              if (vmType.equals(VirtualMachine.Type.NetScalerVm)) {
                  NicTO mgmtNic = vmSpec.getNics()[0];
                  OptionValue option = new OptionValue();
                  option.setKey("machine.id");
                  option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway());
                  extraOptions.add(option);
              }
  
              configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName);
  
              // config video card
              configureVideoCard(vmMo, vmSpec, vmConfigSpec);
  
              setBootOptions(vmSpec, bootMode, vmConfigSpec);
  
              // Config vTPM
              configureVirtualTPM(vmMo, vmSpec, vmConfigSpec);
  
              if (StringUtils.isNotEmpty(vmStoragePolicyId)) {
                  vmConfigSpec.getVmProfile().add(vmProfileSpec);
                  if (logger.isTraceEnabled()) {
                      logger.trace(String.format("Configuring the VM %s with storage policy: %s", vmInternalCSName, vmStoragePolicyId));
                  }
              }
  
              //
              // Configure VM
              //
              if (!vmMo.configureVm(vmConfigSpec)) {
                  throw new Exception("Failed to configure VM before start. vmName: " + vmInternalCSName);
              }
  
              if (vmSpec.getType() == VirtualMachine.Type.DomainRouter) {
                  hyperHost.setRestartPriorityForVM(vmMo, DasVmPriority.HIGH.value());
              }
  
              // Resizing root disk only when explicit requested by user
              final Map<String, String> vmDetails = cmd.getVirtualMachine().getDetails();
              if (!deployAsIs && rootDiskTO != null && !hasSnapshot && (vmDetails != null && vmDetails.containsKey(ApiConstants.ROOT_DISK_SIZE))) {
                  resizeRootDiskOnVMStart(vmMo, rootDiskTO, hyperHost, context);
              }
  
              //
              // Post Configuration
              //
  
              vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMask));
              postNvpConfigBeforeStart(vmMo, vmSpec);
  
              Map<String, Map<String, String>> iqnToData = new HashMap<>();
  
              postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, iqnToData, hyperHost, context);
  
              //
              // Power-on VM
              //
              if (powerOnVM(vmMo, vmInternalCSName, vmNameOnVcenter)) {
                  logger.debug(String.format("VM %s has been started successfully with hostname %s.", vmInternalCSName, vmNameOnVcenter));
              } else {
                  throw new Exception("Failed to start VM. vmName: " + vmInternalCSName + " with hostname " + vmNameOnVcenter);
              }
  
              StartAnswer startAnswer = new StartAnswer(cmd);
  
              startAnswer.setIqnToData(iqnToData);
  
              if (vmSpec.getType() != VirtualMachine.Type.User) {
                  String controlIp = getControlIp(nics);
                  // check if the router is up?
                  for (int count = 0; count < 60; count++) {
                      final boolean result = _vrResource.connect(controlIp, 1, 5000);
                      if (result) {
                          break;
                      }
                  }
  
                  try {
                      String homeDir = System.getProperty("user.home");
                      File pemFile = new File(homeDir + "/.ssh/id_rsa");
                      FileUtil.scpPatchFiles(controlIp, VRScripts.CONFIG_CACHE_LOCATION, DefaultDomRSshPort, pemFile, systemVmPatchFiles, BASEPATH);
                      if (!_vrResource.isSystemVMSetup(vmInternalCSName, controlIp)) {
                          String errMsg = "Failed to patch systemVM";
                          logger.error(errMsg);
                          return new StartAnswer(cmd, errMsg);
                      }
                  } catch (Exception e) {
                      String errMsg = "Failed to scp files to system VM. Patching of systemVM failed";
                      logger.error(errMsg, e);
                      return new StartAnswer(cmd, String.format("%s due to: %s", errMsg, e.getMessage()));
                  }
              }
  
              // Since VM was successfully powered-on, if there was an existing VM in a different cluster that was unregistered, delete all the files associated with it.
              if (existingVmName != null && existingVmFileLayout != null) {
                  List<String> vmDatastoreNames = new ArrayList<>();
                  for (DatastoreMO vmDatastore : vmMo.getAllDatastores()) {
                      vmDatastoreNames.add(vmDatastore.getName());
                  }
                  // Don't delete files that are in a datastore that is being used by the new VM as well (zone-wide datastore).
                  List<String> skipDatastores = new ArrayList<>();
                  for (DatastoreMO existingDatastore : existingDatastores) {
                      if (vmDatastoreNames.contains(existingDatastore.getName())) {
                          skipDatastores.add(existingDatastore.getName());
                      }
                  }
                  deleteUnregisteredVmFiles(existingVmFileLayout, dcMo, true, skipDatastores);
              }
  
              return startAnswer;
          } catch (Throwable e) {
              StartAnswer startAnswer = new StartAnswer(cmd, createLogMessageException(e, cmd));
              if (vmAlreadyExistsInVcenter) {
                  startAnswer.setContextParam("stopRetry", "true");
              }
  
              if (existingVmName != null && existingVmFileInfo != null) {
                  logger.debug(String.format("Since VM start failed, registering back an existing VM: [%s] that was unregistered.", existingVmName));
                  try {
                      DatastoreFile fileInDatastore = new DatastoreFile(existingVmFileInfo.getVmPathName());
                      DatastoreMO existingVmDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName()));
                      registerVm(existingVmName, existingVmDsMo);
                  } catch (Exception ex) {
                      String message = String.format("Failed to register an existing VM: [%s] due to [%s].", existingVmName, VmwareHelper.getExceptionMessage(ex));
                      logger.error(message, ex);
                  }
              }
              return startAnswer;
          }
      }
  
      private Pair<ManagedObjectReference, DatastoreMO> getVolumeDatastoreDetails(DiskTO vol, HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> dataStoresDetails) throws Exception {
          boolean managed = false;
          String iScsiName = null;
          Map<String, String> details = vol.getDetails();
          if (MapUtils.isNotEmpty(details)) {
              managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
              iScsiName = details.get(DiskTO.IQN);
          }
  
          VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
          DataStoreTO primaryStore = volumeTO.getDataStore();
          String primaryStoreUuid = primaryStore.getUuid();
          // if the storage is managed, iScsiName should not be null
          String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStoreUuid;
          Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails = dataStoresDetails.get(datastoreName);
          if (volumeDsDetails == null) {
              throw new Exception("Primary datastore " + primaryStore.getUuid() + " is not mounted on host.");
          }
  
          return volumeDsDetails;
      }
  
      private void syncVolumeDatastoreAndPathForDatastoreCluster(DiskTO vol, VirtualMachineDiskInfoBuilder diskInfoBuilder, VirtualMachineDiskInfo matchingExistingDisk,
                                                                 Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails, List<Pair<Integer, ManagedObjectReference>> diskDatastores,
                                                                 VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
          if (vol.getDetails() == null || vol.getDetails().get(DiskTO.PROTOCOL_TYPE) == null || !vol.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) {
              return;
          }
  
          if (diskInfoBuilder != null && matchingExistingDisk != null) {
              String[] diskChain = matchingExistingDisk.getDiskChain();
              if (diskChain != null && diskChain.length > 0) {
                  DatastoreFile file = new DatastoreFile(diskChain[0]);
                  VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
                  if (!file.getFileBaseName().equalsIgnoreCase(volumeTO.getPath())) {
                      if (logger.isInfoEnabled()) {
                          logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + file.getFileBaseName());
                      }
                      volumeTO.setPath(file.getFileBaseName());
                      vol.setPath(file.getFileBaseName());
                  }
              }
              DatastoreMO diskDatastoreMofromVM = getDataStoreWhereDiskExists(hyperHost, context, diskInfoBuilder, vol, diskDatastores);
              if (diskDatastoreMofromVM != null) {
                  String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID);
                  VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
                  DataStoreTO primaryStore = volumeTO.getDataStore();
                  if (actualPoolUuid != null && !actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) {
                      volumeDsDetails = new Pair<>(diskDatastoreMofromVM.getMor(), diskDatastoreMofromVM);
                      if (logger.isInfoEnabled()) {
                          logger.info("Detected datastore uuid change on volume: " + volumeTO.getId() + " " + primaryStore.getUuid() + " -> " + actualPoolUuid);
                      }
                      ((PrimaryDataStoreTO)primaryStore).setUuid(actualPoolUuid);
                  }
              }
          }
      }
  
      private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception {
          final int retry = 20;
          int retryAttempt = 0;
          while (++retryAttempt <= retry) {
              try {
                  logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt));
                  return vmMo.powerOn();
              } catch (Exception e) {
                  logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e);
                  if (e.getMessage() != null &&
                          (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") ||
                                  e.getMessage().contains("Unable to access file") ||
                                  e.getMessage().contains("it is locked"))) {
                      logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter));
                      try {
                          Thread.sleep(1000);
                      } catch (InterruptedException ie) {
                          logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName));
                      }
                  } else {
                      throw e;
                  }
              }
          }
          return false;
      }
  
      private boolean multipleIsosAtached(DiskTO[] sortedDisks) {
          return Arrays.stream(sortedDisks).filter(disk -> disk.getType() == Volume.Type.ISO).count() > 1;
      }
  
      private DiskTO[] getDisks(DiskTO[] sortedDisks) {
         return Arrays.stream(sortedDisks).filter(vol -> ((vol.getPath() != null &&
                  vol.getPath().contains(ConfigDrive.CONFIGDRIVEDIR))) || (vol.getType() != Volume.Type.ISO)).toArray(DiskTO[]::new);
      }
  
      private void configureIso(VmwareHypervisorHost hyperHost, VirtualMachineMO vmMo, DiskTO vol,
                                VirtualDeviceConfigSpec[] deviceConfigSpecArray, int ideUnitNumber, int i) throws Exception {
          TemplateObjectTO iso = (TemplateObjectTO) vol.getData();
  
          if (iso.getPath() != null && !iso.getPath().isEmpty()) {
              DataStoreTO imageStore = iso.getDataStore();
              if (!(imageStore instanceof NfsTO)) {
                  logger.debug("unsupported protocol");
                  throw new Exception("unsupported protocol");
              }
              NfsTO nfsImageStore = (NfsTO) imageStore;
              String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath();
              Pair<String, ManagedObjectReference> isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath);
              assert (isoDatastoreInfo != null);
              assert (isoDatastoreInfo.second() != null);
  
              deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
              Pair<VirtualDevice, Boolean> isoInfo =
                      VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber, i + 1);
              deviceConfigSpecArray[i].setDevice(isoInfo.first());
              if (isoInfo.second()) {
                  if (logger.isDebugEnabled())
                      logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first()));
                  deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD);
              } else {
                  if (logger.isDebugEnabled())
                      logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
                  deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT);
              }
          }
      }
  
      private String mapAdapterType(String adapterStringFromOVF) {
          if (StringUtils.isBlank(adapterStringFromOVF) || adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.E1000.toString())) {
              return VirtualEthernetCardType.E1000.toString();
          } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.PCNet32.toString())) {
              return VirtualEthernetCardType.PCNet32.toString();
          } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.Vmxnet2.toString())) {
              return VirtualEthernetCardType.Vmxnet2.toString();
          } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.Vmxnet3.toString())) {
              return VirtualEthernetCardType.Vmxnet3.toString();
          }
          return VirtualEthernetCardType.E1000.toString();
      }
  
      private int getDisksChangesNumberFromDisksSpec(DiskTO[] disks, boolean deployAsIs) {
          if (!deployAsIs) {
              return disks.length;
          } else {
              int datadisksNumber = 0;
              if (ArrayUtils.isNotEmpty(disks)) {
                  List<DiskTO> datadisks = Arrays.stream(disks).filter(x -> x.getType() == Volume.Type.DATADISK).collect(Collectors.toList());
                  if (CollectionUtils.isNotEmpty(datadisks)) {
                      datadisksNumber = datadisks.size();
                  }
              }
              return datadisksNumber;
          }
      }
  
      /**
       * Configure VNC
       */
      private void configureVNC(VirtualMachineTO vmSpec, ArrayList<OptionValue> extraOptions, VirtualMachineConfigSpec vmConfigSpec, VmwareHypervisorHost hyperHost, String vmInternalCSName) throws Exception {
          String keyboardLayout = null;
          if (vmSpec.getDetails() != null)
              keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD);
          vmConfigSpec.getExtraConfig()
                  .addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout)));
  
      }
  
      private void ensureDiskControllersInternal(VirtualMachineMO vmMo, Boolean systemVm,
                                                 Pair<String, String> controllerInfo,
                                                 DiskControllerType systemVmScsiControllerType,
                                                 int numScsiControllerForSystemVm,
                                                 int firstScsiControllerBusNum, boolean deployAsIs) throws Exception {
          if (systemVm) {
              ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum);
          } else if (!deployAsIs) {
              ensureDiskControllers(vmMo, controllerInfo);
          }
      }
  
      private void tearDownVmDevices(VirtualMachineMO vmMo, boolean hasSnapshot, boolean deployAsIs) throws Exception {
          if (deployAsIs) {
              vmMo.tearDownDevices(new Class<?>[]{VirtualEthernetCard.class});
          } else if (!hasSnapshot) {
              vmMo.tearDownDevices(new Class<?>[]{VirtualDisk.class, VirtualEthernetCard.class});
          } else {
              vmMo.tearDownDevices(new Class<?>[]{VirtualEthernetCard.class});
          }
      }
  
      private void tearDownVMDisks(VirtualMachineMO vmMo, List<VirtualDisk> disks) throws Exception {
          for (VirtualDisk disk : disks) {
              vmMo.tearDownDevice(disk);
          }
      }
  
      private String getGuestOsIdFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) {
          return translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value();
      }
  
      private Pair<String, String> getControllerInfoFromVmSpec(VirtualMachineTO vmSpec) throws CloudRuntimeException {
          String rootDiskControllerDetail = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER);
          String dataDiskControllerDetail = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER);
          VmwareHelper.validateDiskControllerDetails(rootDiskControllerDetail, dataDiskControllerDetail);
          return new Pair<>(rootDiskControllerDetail, dataDiskControllerDetail);
      }
  
      private String getBootModeFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) {
          String bootMode = null;
          if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) {
              bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE);
          }
          if (bootMode == null) {
              bootMode = ApiConstants.BootType.BIOS.toString();
          }
          return bootMode;
      }
  
      /**
       * Set OVF properties (if available)
       */
      private void setDeployAsIsProperties(VirtualMachineMO vmMo, DeployAsIsInfoTO deployAsIsInfo,
                                           VirtualMachineConfigSpec vmConfigSpec, VmwareHypervisorHost hyperHost) throws Exception {
          if (deployAsIsInfo != null && MapUtils.isNotEmpty(deployAsIsInfo.getProperties())) {
              Map<String, String> properties = deployAsIsInfo.getProperties();
              VmConfigInfo vAppConfig = vmMo.getConfigInfo().getVAppConfig();
              logger.info("Copying OVF properties to the values the user provided");
              setVAppPropertiesToConfigSpec(vAppConfig, properties, vmConfigSpec, hyperHost);
          }
      }
  
      /**
       * Modify the specDisks information to match the cloned VM's disks (from vmMo VM)
       */
      private void mapSpecDisksToClonedDisksAndTearDownDatadisks(VirtualMachineMO vmMo, String vmInternalCSName, DiskTO[] specDisks) {
          try {
              logger.debug("Mapping spec disks information to cloned VM disks for VM " + vmInternalCSName);
              if (vmMo != null && ArrayUtils.isNotEmpty(specDisks)) {
                  List<VirtualDisk> vmDisks = vmMo.getVirtualDisksOrderedByKey();
  
                  List<VirtualDisk> rootDisks = new ArrayList<>();
                  List<DiskTO> sortedRootDisksFromSpec = Arrays.asList(sortVolumesByDeviceId(specDisks))
                          .stream()
                          .filter(x -> x.getType() == Volume.Type.ROOT)
                          .collect(Collectors.toList());
                  for (int i = 0; i < sortedRootDisksFromSpec.size(); i++) {
                      DiskTO specDisk = sortedRootDisksFromSpec.get(i);
                      VirtualDisk vmDisk = vmDisks.get(i);
                      DataTO dataVolume = specDisk.getData();
                      if (dataVolume instanceof VolumeObjectTO) {
                          VolumeObjectTO volumeObjectTO = (VolumeObjectTO) dataVolume;
                          if (!volumeObjectTO.getSize().equals(vmDisk.getCapacityInBytes())) {
                              logger.info("Mapped disk size is not the same as the cloned VM disk size: " +
                                      volumeObjectTO.getSize() + " - " + vmDisk.getCapacityInBytes());
                          }
                          VirtualDeviceBackingInfo backingInfo = vmDisk.getBacking();
                          if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) {
                              VirtualDiskFlatVer2BackingInfo backing = (VirtualDiskFlatVer2BackingInfo) backingInfo;
                              String fileName = backing.getFileName();
                              if (StringUtils.isNotBlank(fileName)) {
                                  String[] fileNameParts = fileName.split(" ");
                                  String datastoreUuid = fileNameParts[0].replace("[", "").replace("]", "");
                                  String relativePath = fileNameParts[1].split("/")[1].replace(".vmdk", "");
                                  String vmSpecDatastoreUuid = volumeObjectTO.getDataStore().getUuid().replaceAll("-", "");
                                  if (!datastoreUuid.equals(vmSpecDatastoreUuid)) {
                                      logger.info("Mapped disk datastore UUID is not the same as the cloned VM datastore UUID: " +
                                              datastoreUuid + " - " + vmSpecDatastoreUuid);
                                  }
                                  volumeObjectTO.setPath(relativePath);
                                  specDisk.setPath(relativePath);
                                  rootDisks.add(vmDisk);
                              } else {
                                  logger.error("Empty backing filename for volume " + volumeObjectTO.getName());
                              }
                          } else {
                              logger.error("Could not get volume backing info for volume " + volumeObjectTO.getName());
                          }
                      }
                  }
                  vmDisks.removeAll(rootDisks);
                  if (CollectionUtils.isNotEmpty(vmDisks)) {
                      logger.info("Tearing down datadisks for deploy-as-is VM");
                      tearDownVMDisks(vmMo, vmDisks);
                  }
              }
          } catch (Exception e) {
              String msg = "Error mapping deploy-as-is VM disks from cloned VM " + vmInternalCSName;
              logger.error(msg, e);
              throw new CloudRuntimeException(e);
          }
      }
  
      private void setBootOptions(VirtualMachineTO vmSpec, String bootMode, VirtualMachineConfigSpec vmConfigSpec) {
          VirtualMachineBootOptions bootOptions = null;
          if (StringUtils.isNotBlank(bootMode) && !bootMode.equalsIgnoreCase("bios")) {
              vmConfigSpec.setFirmware("efi");
              if (vmSpec.getDetails().containsKey(ApiConstants.BootType.UEFI.toString()) && "secure".equalsIgnoreCase(vmSpec.getDetails().get(ApiConstants.BootType.UEFI.toString()))) {
                  if (bootOptions == null) {
                      bootOptions = new VirtualMachineBootOptions();
                  }
                  bootOptions.setEfiSecureBootEnabled(true);
              }
          }
          if (vmSpec.isEnterHardwareSetup()) {
              if (bootOptions == null) {
                  bootOptions = new VirtualMachineBootOptions();
              }
              if (logger.isDebugEnabled()) {
                  logger.debug(String.format("configuring VM '%s' to enter hardware setup",vmSpec.getName()));
              }
              bootOptions.setEnterBIOSSetup(vmSpec.isEnterHardwareSetup());
          }
          if (bootOptions != null) {
              vmConfigSpec.setBootOptions(bootOptions);
          }
      }
  
      /**
       * Set the ovf section spec from existing vApp configuration
       */
      protected List<VAppOvfSectionSpec> copyVAppConfigOvfSectionFromOVF(VmConfigInfo vAppConfig, boolean useEdit) {
          List<VAppOvfSectionInfo> ovfSection = vAppConfig.getOvfSection();
          List<VAppOvfSectionSpec> specs = new ArrayList<>();
          for (VAppOvfSectionInfo info : ovfSection) {
              VAppOvfSectionSpec spec = new VAppOvfSectionSpec();
              spec.setInfo(info);
              spec.setOperation(useEdit ? ArrayUpdateOperation.EDIT : ArrayUpdateOperation.ADD);
              specs.add(spec);
          }
          return specs;
      }
  
      private Map<String, Pair<String, Boolean>> getOVFMap(List<OVFPropertyTO> props) {
          Map<String, Pair<String, Boolean>> map = new HashMap<>();
          for (OVFPropertyTO prop : props) {
              String value = getPropertyValue(prop);
              Pair<String, Boolean> pair = new Pair<>(value, prop.isPassword());
              map.put(prop.getKey(), pair);
          }
          return map;
      }
  
      private String getPropertyValue(OVFPropertyTO prop) {
          String type = prop.getType();
          String value = prop.getValue();
          if ("boolean".equalsIgnoreCase(type)) {
              value = Boolean.parseBoolean(value) ? "True" : "False";
          }
          return value;
      }
  
      /**
       * Set the properties section from existing vApp configuration and values set on ovfProperties
       */
      protected List<VAppPropertySpec> copyVAppConfigPropertySectionFromOVF(VmConfigInfo vAppConfig, Map<String, String> ovfProperties,
                                                                            boolean useEdit) {
          List<VAppPropertyInfo> productFromOvf = vAppConfig.getProperty();
          List<VAppPropertySpec> specs = new ArrayList<>();
          for (VAppPropertyInfo info : productFromOvf) {
              VAppPropertySpec spec = new VAppPropertySpec();
              if (ovfProperties.containsKey(info.getId())) {
                  String value = ovfProperties.get(info.getId());
                  info.setValue(value);
                  logger.info("Setting OVF property ID = " + info.getId() + " VALUE = " + value);
              }
              spec.setInfo(info);
              spec.setOperation(useEdit ? ArrayUpdateOperation.EDIT : ArrayUpdateOperation.ADD);
              specs.add(spec);
          }
          return specs;
      }
  
      /**
       * Set the product section spec from existing vApp configuration
       */
      protected List<VAppProductSpec> copyVAppConfigProductSectionFromOVF(VmConfigInfo vAppConfig, boolean useEdit) {
          List<VAppProductInfo> productFromOvf = vAppConfig.getProduct();
          List<VAppProductSpec> specs = new ArrayList<>();
          for (VAppProductInfo info : productFromOvf) {
              VAppProductSpec spec = new VAppProductSpec();
              spec.setInfo(info);
              logger.info("Procuct info KEY " + info.getKey());
              spec.setOperation(useEdit ? ArrayUpdateOperation.EDIT : ArrayUpdateOperation.ADD);
              specs.add(spec);
          }
          return specs;
      }
  
      /**
       * Set the vApp configuration to vmConfig spec, copying existing configuration from vAppConfig
       * and seting properties values from ovfProperties
       */
      protected void setVAppPropertiesToConfigSpec(VmConfigInfo vAppConfig,
                                                   Map<String, String> ovfProperties,
                                                   VirtualMachineConfigSpec vmConfig, VmwareHypervisorHost hyperHost) throws Exception {
          VmConfigSpec vmConfigSpec = new VmConfigSpec();
          vmConfigSpec.getEula().addAll(vAppConfig.getEula());
          vmConfigSpec.setInstallBootStopDelay(vAppConfig.getInstallBootStopDelay());
          vmConfigSpec.setInstallBootRequired(vAppConfig.isInstallBootRequired());
          vmConfigSpec.setIpAssignment(vAppConfig.getIpAssignment());
          vmConfigSpec.getOvfEnvironmentTransport().addAll(vAppConfig.getOvfEnvironmentTransport());
  
          // For backward compatibility, prior to Vmware 6.5 use EDIT operation instead of ADD
          boolean useEditOperation = hyperHost.getContext().getServiceContent().getAbout().getApiVersion().compareTo("6.5") < 1;
          vmConfigSpec.getProduct().addAll(copyVAppConfigProductSectionFromOVF(vAppConfig, useEditOperation));
          vmConfigSpec.getProperty().addAll(copyVAppConfigPropertySectionFromOVF(vAppConfig, ovfProperties, useEditOperation));
          vmConfigSpec.getOvfSection().addAll(copyVAppConfigOvfSectionFromOVF(vAppConfig, useEditOperation));
          vmConfig.setVAppConfig(vmConfigSpec);
      }
  
      private String appendFileType(String path, String fileType) {
          if (StringUtils.isBlank(path)) {
              throw new CloudRuntimeException("No path given, cannot append filetype " + fileType);
          }
          if (fileType == null || path.toLowerCase().endsWith(fileType.toLowerCase())) {
              return path;
          }
  
          return path + fileType;
      }
  
      private void resizeRootDiskOnVMStart(VirtualMachineMO vmMo, DiskTO rootDiskTO, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
          final Pair<VirtualDisk, String> vdisk = getVirtualDiskInfo(vmMo, appendFileType(rootDiskTO.getPath(), VMDK_EXTENSION));
          assert (vdisk != null);
  
          long reqSize = 0L;
          final VolumeObjectTO volumeTO = ((VolumeObjectTO) rootDiskTO.getData());
          if (volumeTO != null) {
              reqSize = volumeTO.getSize() / 1024;
          }
          final VirtualDisk disk = vdisk.first();
          if (reqSize > disk.getCapacityInKB()) {
              final VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(vmMo.getDiskInfoBuilder(), rootDiskTO, hyperHost, context);
              assert (diskInfo != null);
              final String[] diskChain = diskInfo.getDiskChain();
  
              if (diskChain != null && diskChain.length > 1) {
                  logger.warn("Disk chain length for the VM is greater than one, this is not supported");
                  throw new CloudRuntimeException("Unsupported VM disk chain length: " + diskChain.length);
              }
  
              boolean resizingSupported = false;
              String deviceBusName = diskInfo.getDiskDeviceBusName();
              if (deviceBusName != null && (deviceBusName.toLowerCase().contains("scsi") || deviceBusName.toLowerCase().contains("lsi"))) {
                  resizingSupported = true;
              }
              if (!resizingSupported) {
                  logger.warn("Resizing of root disk is only support for scsi device/bus, the provide VM's disk device bus name is " + diskInfo.getDiskDeviceBusName());
                  throw new CloudRuntimeException("Unsupported VM root disk device bus: " + diskInfo.getDiskDeviceBusName());
              }
  
              disk.setCapacityInKB(reqSize);
              VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
              VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
              deviceConfigSpec.setDevice(disk);
              deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.EDIT);
              vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
              if (!vmMo.configureVm(vmConfigSpec)) {
                  throw new Exception("Failed to configure VM for given root disk size. vmName: " + vmMo.getName());
              }
          }
      }
  
  
      /**
       * Generate the mac sequence from the nics.
       */
      protected String generateMacSequence(NicTO[] nics) {
          if (nics.length == 0) {
              return "";
          }
  
          StringBuffer sbMacSequence = new StringBuffer();
          for (NicTO nicTo : sortNicsByDeviceId(nics)) {
              sbMacSequence.append(nicTo.getMac()).append("|");
          }
          if (!sbMacSequence.toString().isEmpty()) {
              sbMacSequence.deleteCharAt(sbMacSequence.length() - 1); //Remove extra '|' char appended at the end
          }
  
          return sbMacSequence.toString();
      }
  
      /**
       * Update boot args with the new nic mac addresses.
       */
      protected String replaceNicsMacSequenceInBootArgs(String oldMacSequence, String newMacSequence, VirtualMachineTO vmSpec) {
          String bootArgs = vmSpec.getBootArgs();
          if (StringUtils.isNoneBlank(bootArgs, oldMacSequence, newMacSequence)) {
              return bootArgs.replace(oldMacSequence, newMacSequence);
          }
          return "";
      }
  
      /**
       * Sets video card memory to the one provided in detail svga.vramSize (if provided) on {@code vmConfigSpec}.
       * 64MB was always set before.
       * Size must be in KB.
       *
       * @param vmMo         virtual machine mo
       * @param vmSpec       virtual machine specs
       * @param vmConfigSpec virtual machine config spec
       * @throws Exception exception
       */
      protected void configureVideoCard(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, VirtualMachineConfigSpec vmConfigSpec) throws Exception {
          if (vmSpec.getDetails().containsKey(VmDetailConstants.SVGA_VRAM_SIZE)) {
              String value = vmSpec.getDetails().get(VmDetailConstants.SVGA_VRAM_SIZE);
              try {
                  long svgaVmramSize = Long.parseLong(value);
                  setNewVRamSizeVmVideoCard(vmMo, svgaVmramSize, vmConfigSpec);
              } catch (NumberFormatException e) {
                  logger.error("Unexpected value, cannot parse " + value + " to long due to: " + e.getMessage());
              }
          }
      }
  
      /**
       * Search for vm video card iterating through vm device list
       *
       * @param vmMo          virtual machine mo
       * @param svgaVmramSize new svga vram size (in KB)
       * @param vmConfigSpec  virtual machine config spec
       */
      protected void setNewVRamSizeVmVideoCard(VirtualMachineMO vmMo, long svgaVmramSize, VirtualMachineConfigSpec vmConfigSpec) throws Exception {
          for (VirtualDevice device : vmMo.getAllDeviceList()) {
              if (device instanceof VirtualMachineVideoCard) {
                  VirtualMachineVideoCard videoCard = (VirtualMachineVideoCard) device;
                  modifyVmVideoCardVRamSize(videoCard, vmMo, svgaVmramSize, vmConfigSpec);
              }
          }
      }
  
      /**
       * Modifies vm vram size if it was set to a different size to the one provided in svga.vramSize (vm_instance_details or template_vm_details) on {@code vmConfigSpec}
       *
       * @param videoCard     vm's video card device
       * @param vmMo          virtual machine mo
       * @param svgaVmramSize new svga vram size (in KB)
       * @param vmConfigSpec  virtual machine config spec
       */
      protected void modifyVmVideoCardVRamSize(VirtualMachineVideoCard videoCard, VirtualMachineMO vmMo, long svgaVmramSize, VirtualMachineConfigSpec vmConfigSpec) {
          if (videoCard.getVideoRamSizeInKB() != svgaVmramSize) {
             logger.info("Video card memory was set " + toHumanReadableSize(videoCard.getVideoRamSizeInKB()) + " instead of " + toHumanReadableSize(svgaVmramSize));
              configureSpecVideoCardNewVRamSize(videoCard, svgaVmramSize, vmConfigSpec);
          }
      }
  
      /**
       * Add edit spec on {@code vmConfigSpec} to modify svga vram size
       *
       * @param videoCard     video card device to edit providing the svga vram size
       * @param svgaVmramSize new svga vram size (in KB)
       * @param vmConfigSpec  virtual machine spec
       */
      protected void configureSpecVideoCardNewVRamSize(VirtualMachineVideoCard videoCard, long svgaVmramSize, VirtualMachineConfigSpec vmConfigSpec) {
          videoCard.setVideoRamSizeInKB(svgaVmramSize);
          videoCard.setUseAutoDetect(false);
  
          VirtualDeviceConfigSpec arrayVideoCardConfigSpecs = new VirtualDeviceConfigSpec();
          arrayVideoCardConfigSpecs.setDevice(videoCard);
          arrayVideoCardConfigSpecs.setOperation(VirtualDeviceConfigSpecOperation.EDIT);
  
          vmConfigSpec.getDeviceChange().add(arrayVideoCardConfigSpecs);
      }
  
      /**
       * Add or Remove virtual TPM module
       *
       * @param vmMo         virtual machine mo
       * @param vmSpec       virtual machine specs
       * @param vmConfigSpec virtual machine config spec
       * @throws Exception exception
       */
      protected void configureVirtualTPM(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, VirtualMachineConfigSpec vmConfigSpec) throws Exception {
          String virtualTPMEnabled = vmSpec.getDetails().getOrDefault(VmDetailConstants.VIRTUAL_TPM_ENABLED, null);
          if (Boolean.parseBoolean(virtualTPMEnabled)) {
              for (VirtualDevice device : vmMo.getAllDeviceList()) {
                  if (device instanceof VirtualTPM) {
                      logger.debug(String.format("Virtual TPM device has already been added to VM %s, returning", vmMo.getVmName()));
                      return;
                  }
              }
              logger.debug(String.format("Adding Virtual TPM device to the VM %s", vmMo.getVmName()));
              addVirtualTPMDevice(vmConfigSpec);
          } else if (virtualTPMEnabled == null) {
              logger.debug(String.format("Virtual TPM device is neither enabled nor disabled for VM %s, skipping", vmMo.getVmName()));
          } else {
              logger.debug(String.format("Virtual TPM device is disabled for VM %s", vmMo.getVmName()));
              for (VirtualDevice device : vmMo.getAllDeviceList()) {
                  if (device instanceof VirtualTPM) {
                      logger.debug(String.format("Removing Virtual TPM device from VM %s as it is disabled", vmMo.getVmName()));
                      removeVirtualTPMDevice(vmConfigSpec, (VirtualTPM) device);
                  }
              }
          }
      }
  
      protected void addVirtualTPMDevice(VirtualMachineConfigSpec vmConfigSpec) {
          Description description = new Description();
          description.setSummary("Trusted Platform Module");
          description.setLabel("Trusted Platform Module");
          VirtualTPM virtualTPM = new VirtualTPM();
          virtualTPM.setDeviceInfo(description);
          VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
          deviceConfigSpec.setDevice(virtualTPM);
          deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD);
          vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
      }
  
      protected void removeVirtualTPMDevice(VirtualMachineConfigSpec vmConfigSpec, VirtualTPM virtualTPM) {
          VirtualDeviceConfigSpec virtualDeviceConfigSpec = new VirtualDeviceConfigSpec();
          virtualDeviceConfigSpec.setDevice(virtualTPM);
          virtualDeviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.REMOVE);
          vmConfigSpec.getDeviceChange().add(virtualDeviceConfigSpec);
      }
  
      private void tearDownVm(VirtualMachineMO vmMo) throws Exception {
  
          if (vmMo == null)
              return;
  
          boolean hasSnapshot;
          hasSnapshot = vmMo.hasSnapshot();
          if (!hasSnapshot)
              vmMo.tearDownDevices(new Class<?>[]{VirtualDisk.class, VirtualEthernetCard.class});
          else
              vmMo.tearDownDevices(new Class<?>[]{VirtualEthernetCard.class});
          vmMo.ensureScsiDeviceController();
      }
  
      int getReservedMemoryMb(VirtualMachineTO vmSpec) {
          if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) {
              if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) {
                  float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION)));
                  return (int) (reservedMemory / ResourceType.bytesToMiB);
              }
              return (int) (vmSpec.getMinRam() / ResourceType.bytesToMiB);
          }
          return 0;
      }
  
      int getReservedCpuMHZ(VirtualMachineTO vmSpec) {
          if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveCpu.key()).equalsIgnoreCase("true")) {
              return vmSpec.getMinSpeed() * vmSpec.getCpus();
          }
          return 0;
      }
  
      // return the finalized disk chain for startup, from top to bottom
      private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, DiskTO vol, VirtualMachineDiskInfo diskInfo,
                                     DatastoreMO dsMo) throws Exception {
  
          VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
          Map<String, String> details = vol.getDetails();
          boolean isManaged = false;
  
          if (details != null) {
              isManaged = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
          }
  
          String datastoreDiskPath;
          if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
              datastoreDiskPath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk");
              if (!dsMo.fileExists(datastoreDiskPath)) {
                  datastoreDiskPath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmMo.getName(), volumeTO.getPath() + ".vmdk");
              }
              if (!dsMo.fileExists(datastoreDiskPath)) {
                  datastoreDiskPath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null);
              }
          } else {
              // we will honor vCenter's meta if it exists
              if (diskInfo != null) {
                  // to deal with run-time upgrade to maintain the new datastore folder structure
                  String disks[] = diskInfo.getDiskChain();
                  for (int i = 0; i < disks.length; i++) {
                      DatastoreFile file = new DatastoreFile(disks[i]);
                      if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) {
                          logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder");
                          disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value());
                      }
                  }
                  return disks;
              }
  
              if (isManaged) {
                  String vmdkPath = new DatastoreFile(volumeTO.getPath()).getFileBaseName();
  
                  if (volumeTO.getVolumeType() == Volume.Type.ROOT) {
                      if (vmdkPath == null) {
                          vmdkPath = volumeTO.getName();
                      }
  
                      datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, vmdkPath);
                  } else {
                      if (vmdkPath == null) {
                          vmdkPath = dsMo.getName();
                      }
  
                      datastoreDiskPath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION);
                  }
              } else {
                  datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value());
              }
          }
          if (!dsMo.fileExists(datastoreDiskPath)) {
              logger.warn("Volume " + volumeTO.getId() + " does not seem to exist on datastore, out of sync? path: " + datastoreDiskPath);
          }
  
          return new String[]{datastoreDiskPath};
      }
  
      // Pair<internal CS name, vCenter display name>
      private Pair<String, String> composeVmNames(VirtualMachineTO vmSpec) {
          String vmInternalCSName = vmSpec.getName();
          String vmNameOnVcenter = vmSpec.getName();
          String hostNameInDetails = null;
          if (_instanceNameFlag && MapUtils.isNotEmpty(vmSpec.getDetails()) && vmSpec.getDetails().containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) {
              hostNameInDetails = vmSpec.getDetails().get(VmDetailConstants.NAME_ON_HYPERVISOR);
          }
          if (StringUtils.isNotBlank(hostNameInDetails)) {
              vmNameOnVcenter = hostNameInDetails;
          } else if (_instanceNameFlag && vmSpec.getHostName() != null) {
              vmNameOnVcenter = vmSpec.getHostName();
          }
          return new Pair<>(vmInternalCSName, vmNameOnVcenter);
      }
  
      protected void configNestedHVSupport(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, VirtualMachineConfigSpec vmConfigSpec) throws Exception {
  
          VmwareContext context = vmMo.getContext();
          if ("true".equals(vmSpec.getDetails().get(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG))) {
              if (logger.isDebugEnabled())
                  logger.debug("Nested Virtualization enabled in configuration, checking hypervisor capability");
  
              ManagedObjectReference hostMor = vmMo.getRunningHost().getMor();
              ManagedObjectReference computeMor = context.getVimClient().getMoRefProp(hostMor, "parent");
              ManagedObjectReference environmentBrowser = context.getVimClient().getMoRefProp(computeMor, "environmentBrowser");
              HostCapability hostCapability = context.getService().queryTargetCapabilities(environmentBrowser, hostMor);
              Boolean nestedHvSupported = hostCapability.isNestedHVSupported();
              if (nestedHvSupported == null) {
                  // nestedHvEnabled property is supported only since VMware 5.1. It's not defined for earlier versions.
                  logger.warn("Hypervisor doesn't support nested virtualization, unable to set config for VM " + vmSpec.getName());
              } else if (nestedHvSupported) {
                  logger.debug("Hypervisor supports nested virtualization, enabling for VM " + vmSpec.getName());
                  vmConfigSpec.setNestedHVEnabled(true);
              } else {
                  logger.warn("Hypervisor doesn't support nested virtualization, unable to set config for VM " + vmSpec.getName());
                  vmConfigSpec.setNestedHVEnabled(false);
              }
          }
      }
  
      private static void configBasicExtraOption(List<OptionValue> extraOptions, VirtualMachineTO vmSpec) {
          OptionValue newVal = new OptionValue();
          newVal.setKey("machine.id");
          newVal.setValue(vmSpec.getBootArgs());
          extraOptions.add(newVal);
  
          newVal = new OptionValue();
          newVal.setKey("devices.hotplug");
          newVal.setValue("true");
          extraOptions.add(newVal);
      }
  
      private static void configNvpExtraOption(List<OptionValue> extraOptions, VirtualMachineTO vmSpec, Map<String, String> nicUuidToDvSwitchUuid) {
          /**
           * Extra Config : nvp.vm-uuid = uuid
           *  - Required for Nicira NVP integration
           */
          OptionValue newVal = new OptionValue();
          newVal.setKey("nvp.vm-uuid");
          newVal.setValue(vmSpec.getUuid());
          extraOptions.add(newVal);
  
          /**
           * Extra Config : nvp.iface-id.<num> = uuid
           *  - Required for Nicira NVP integration
           */
          int nicNum = 0;
          for (NicTO nicTo : sortNicsByDeviceId(vmSpec.getNics())) {
              if (nicTo.getUuid() != null) {
                  newVal = new OptionValue();
                  newVal.setKey("nvp.iface-id." + nicNum);
                  newVal.setValue(nicTo.getUuid());
                  extraOptions.add(newVal);
              }
              nicNum++;
          }
      }
  
      private static void configCustomExtraOption(List<OptionValue> extraOptions, VirtualMachineTO vmSpec) {
          // we no longer to validation anymore
          for (Map.Entry<String, String> entry : vmSpec.getDetails().entrySet()) {
              if (entry.getKey().equalsIgnoreCase(VmDetailConstants.BOOT_MODE)) {
                  continue;
              }
              OptionValue newVal = new OptionValue();
              newVal.setKey(entry.getKey());
              newVal.setValue(entry.getValue());
              extraOptions.add(newVal);
          }
      }
  
      private void postNvpConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec) throws Exception {
          /**
           * We need to configure the port on the DV switch after the host is
           * connected. So make this happen between the configure and start of
           * the VM
           */
          int nicIndex = 0;
          for (NicTO nicTo : sortNicsByDeviceId(vmSpec.getNics())) {
              if (nicTo.getBroadcastType() == BroadcastDomainType.Lswitch) {
                  // We need to create a port with a unique vlan and pass the key to the nic device
                  logger.trace("Nic " + nicTo + " is connected to an NVP logicalswitch");
                  VirtualDevice nicVirtualDevice = vmMo.getNicDeviceByIndex(nicIndex);
                  if (nicVirtualDevice == null) {
                      throw new Exception("Failed to find a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad
                  }
                  VirtualDeviceBackingInfo backing = nicVirtualDevice.getBacking();
                  if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) {
                      // This NIC is connected to a Distributed Virtual Switch
                      VirtualEthernetCardDistributedVirtualPortBackingInfo portInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing;
                      DistributedVirtualSwitchPortConnection port = portInfo.getPort();
                      String portKey = port.getPortKey();
                      String portGroupKey = port.getPortgroupKey();
                      String dvSwitchUuid = port.getSwitchUuid();
  
                      logger.debug("NIC " + nicTo + " is connected to dvSwitch " + dvSwitchUuid + " pg " + portGroupKey + " port " + portKey);
  
                      ManagedObjectReference dvSwitchManager = vmMo.getContext().getVimClient().getServiceContent().getDvSwitchManager();
                      ManagedObjectReference dvSwitch = vmMo.getContext().getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid);
  
                      // Get all ports
                      DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria();
                      criteria.setInside(true);
                      criteria.getPortgroupKey().add(portGroupKey);
                      List<DistributedVirtualPort> dvPorts = vmMo.getContext().getVimClient().getService().fetchDVPorts(dvSwitch, criteria);
  
                      DistributedVirtualPort vmDvPort = null;
                      List<Integer> usedVlans = new ArrayList<>();
                      for (DistributedVirtualPort dvPort : dvPorts) {
                          // Find the port for this NIC by portkey
                          if (portKey.equals(dvPort.getKey())) {
                              vmDvPort = dvPort;
                          }
                          VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort.getConfig().getSetting();
                          VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan();
                          logger.trace("Found port " + dvPort.getKey() + " with vlan " + vlanId.getVlanId());
                          if (vlanId.getVlanId() > 0 && vlanId.getVlanId() < 4095) {
                              usedVlans.add(vlanId.getVlanId());
                          }
                      }
  
                      if (vmDvPort == null) {
                          throw new Exception("Empty port list from dvSwitch for nic " + nicTo);
                      }
  
                      DVPortConfigInfo dvPortConfigInfo = vmDvPort.getConfig();
                      VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPortConfigInfo.getSetting();
  
                      VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan();
                      BoolPolicy blocked = settings.getBlocked();
                      if (blocked.isValue() == Boolean.TRUE) {
                          logger.trace("Port is blocked, set a vlanid and unblock");
                          DVPortConfigSpec dvPortConfigSpec = new DVPortConfigSpec();
                          VMwareDVSPortSetting edittedSettings = new VMwareDVSPortSetting();
                          // Unblock
                          blocked.setValue(Boolean.FALSE);
                          blocked.setInherited(Boolean.FALSE);
                          edittedSettings.setBlocked(blocked);
                          // Set vlan
                          int i;
                          for (i = 1; i < 4095; i++) {
                              if (!usedVlans.contains(i))
                                  break;
                          }
                          vlanId.setVlanId(i); // FIXME should be a determined
                          // based on usage
                          vlanId.setInherited(false);
                          edittedSettings.setVlan(vlanId);
  
                          dvPortConfigSpec.setSetting(edittedSettings);
                          dvPortConfigSpec.setOperation("edit");
                          dvPortConfigSpec.setKey(portKey);
                          List<DVPortConfigSpec> dvPortConfigSpecs = new ArrayList<>();
                          dvPortConfigSpecs.add(dvPortConfigSpec);
                          ManagedObjectReference task = vmMo.getContext().getVimClient().getService().reconfigureDVPortTask(dvSwitch, dvPortConfigSpecs);
                          if (!vmMo.getContext().getVimClient().waitForTask(task)) {
                              throw new Exception("Failed to configure the dvSwitch port for nic " + nicTo);
                          }
                          logger.debug("NIC " + nicTo + " connected to vlan " + i);
                      } else {
                          logger.trace("Port already configured and set to vlan " + vlanId.getVlanId());
                      }
                  } else if (backing instanceof VirtualEthernetCardNetworkBackingInfo) {
                      // This NIC is connected to a Virtual Switch
                      // Nothing to do
                  } else if (backing instanceof VirtualEthernetCardOpaqueNetworkBackingInfo) {
                      //if NSX API VERSION >= 4.2, connect to br-int (nsx.network), do not create portgroup else previous behaviour
                      //OK, connected to OpaqueNetwork
                  } else {
                      logger.error("nic device backing is of type " + backing.getClass().getName());
                      throw new Exception("Incompatible backing for a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad
                  }
              }
              nicIndex++;
          }
      }
      private VirtualMachineDiskInfo getMatchingExistingDiskWithVolumeDetails(VirtualMachineDiskInfoBuilder diskInfoBuilder, String volumePath,
                                                                               String chainInfo, boolean isManaged, String iScsiName, String datastoreUUID,
                                                                               VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
  
          Pair<String, String> dsNameAndFileName = getVMDiskInfo(volumePath, isManaged, iScsiName, datastoreUUID, hyperHost, context);
          String dsName = dsNameAndFileName.first();
          String diskBackingFileBaseName = dsNameAndFileName.second();
  
          VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(diskBackingFileBaseName, dsName);
          if (diskInfo != null) {
              logger.info("Found existing disk info from volume path: " + volumePath);
              return diskInfo;
          } else {
              if (chainInfo != null) {
                  VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class);
                  if (infoInChain != null) {
                      String[] disks = infoInChain.getDiskChain();
                      if (disks.length > 0) {
                          for (String diskPath : disks) {
                              DatastoreFile file = new DatastoreFile(diskPath);
                              diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName);
                              if (diskInfo != null) {
                                  logger.info("Found existing disk from chain info: " + diskPath);
                                  return diskInfo;
                              }
                          }
                      }
  
                      if (diskInfo == null) {
                          diskInfo = diskInfoBuilder.getDiskInfoByDeviceBusName(infoInChain.getDiskDeviceBusName());
                          if (diskInfo != null) {
                              logger.info("Found existing disk from chain device bus information: " + infoInChain.getDiskDeviceBusName());
                              return diskInfo;
                          }
                      }
                  }
              }
          }
          return null;
      }
  
      private Pair<String, String> getVMDiskInfo(String volumePath, boolean isManaged, String iScsiName, String datastoreUUID,
                                                 VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
          String dsName;
          String diskBackingFileBaseName;
  
          if (isManaged) {
              // if the storage is managed, iScsiName should not be null
              dsName = VmwareResource.getDatastoreName(iScsiName);
              diskBackingFileBaseName = new DatastoreFile(volumePath).getFileBaseName();
          } else {
              ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, datastoreUUID);
              DatastoreMO dsMo = new DatastoreMO(context, morDs);
              dsName = dsMo.getName();
              diskBackingFileBaseName = volumePath;
          }
  
          return new Pair<>(dsName, diskBackingFileBaseName);
      }
  
      private VirtualMachineDiskInfo getMatchingExistingDisk(VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO vol, VmwareHypervisorHost hyperHost, VmwareContext context)
              throws Exception {
          if (diskInfoBuilder == null) {
              return null;
          }
  
          VolumeObjectTO volume = (VolumeObjectTO) vol.getData();
          String chainInfo = volume.getChainInfo();
          Map<String, String> details = vol.getDetails();
          boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED));
          String iScsiName = details.get(DiskTO.IQN);
          String datastoreUUID = volume.getDataStore().getUuid();
  
          return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context);
      }
  
      private String getDiskController(VirtualMachineMO vmMo, VirtualMachineDiskInfo matchingExistingDisk, DiskTO vol, Pair<String, String> controllerInfo, boolean deployAsIs) throws Exception {
          DiskControllerType controllerType = DiskControllerType.none;
          if (deployAsIs && matchingExistingDisk != null) {
              String currentBusName = matchingExistingDisk.getDiskDeviceBusName();
              if (currentBusName != null) {
                  logger.info("Chose disk controller based on existing information: " + currentBusName);
                  if (currentBusName.startsWith("ide")) {
                      controllerType = DiskControllerType.ide;
                  } else if (currentBusName.startsWith("scsi")) {
                      controllerType = DiskControllerType.scsi;
                  }
              }
              if (controllerType == DiskControllerType.scsi || controllerType == DiskControllerType.none) {
                  Ternary<Integer, Integer, DiskControllerType> vmScsiControllerInfo = vmMo.getScsiControllerInfo();
                  controllerType = vmScsiControllerInfo.third();
              }
              return controllerType.toString();
          }
  
          return VmwareHelper.getControllerBasedOnDiskType(controllerInfo, vol);
      }
  
      private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks,
                                             Map<String, Map<String, String>> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
          VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
  
          for (DiskTO vol : sortedDisks) {
              if (vol.getType() == Volume.Type.ISO)
                  continue;
  
              VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context);
              if (diskInfo == null) {
                  continue;
              }
  
              String[] diskChain = diskInfo.getDiskChain();
              if (diskChain.length <= 0) {
                  continue;
              }
  
              DatastoreFile file = new DatastoreFile(diskChain[0]);
  
              boolean managed = false;
              Map<String, String> details = vol.getDetails();
              if (details != null) {
                  managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
              }
  
              VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
  
              if (managed) {
                  DatastoreFile originalFile = new DatastoreFile(volumeTO.getPath());
                  if (!file.getFileBaseName().equalsIgnoreCase(originalFile.getFileBaseName())) {
                      if (logger.isInfoEnabled())
                          logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + diskChain[0]);
                  }
              } else {
                  if (!file.getFileBaseName().equalsIgnoreCase(volumeTO.getPath())) {
                      if (logger.isInfoEnabled())
                          logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + file.getFileBaseName());
                  }
              }
  
              VolumeObjectTO volInSpec = getVolumeInSpec(vmSpec, volumeTO);
              if (volInSpec != null) {
                  if (managed) {
                      Map<String, String> data = new HashMap<>();
  
                      String datastoreVolumePath = diskChain[0];
  
                      data.put(StartAnswer.PATH, datastoreVolumePath);
                      data.put(StartAnswer.IMAGE_FORMAT, Storage.ImageFormat.OVA.toString());
  
                      iqnToData.put(details.get(DiskTO.IQN), data);
  
                      vol.setPath(datastoreVolumePath);
                      volumeTO.setPath(datastoreVolumePath);
                      volInSpec.setPath(datastoreVolumePath);
                  } else {
                      volInSpec.setPath(file.getFileBaseName());
                      if (vol.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && vol.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) {
                          volInSpec.setUpdatedDataStoreUUID(volumeTO.getDataStore().getUuid());
                      }
                  }
                  volInSpec.setChainInfo(_gson.toJson(diskInfo));
              }
          }
      }
  
      private void checkAndDeleteDatastoreFile(String filePath, List<String> skipDatastores, DatastoreMO dsMo, DatacenterMO dcMo) throws Exception {
          if (dsMo != null && dcMo != null && (skipDatastores == null || !skipDatastores.contains(dsMo.getName()))) {
              logger.debug("Deleting file: " + filePath);
              dsMo.deleteFile(filePath, dcMo.getMor(), true);
          }
      }
  
      private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo, boolean deleteDisks, List<String> skipDatastores) throws Exception {
          logger.debug("Deleting files associated with an existing VM that was unregistered");
          DatastoreFile vmFolder = null;
          try {
              List<VirtualMachineFileLayoutExFileInfo> fileInfo = vmFileLayout.getFile();
              for (VirtualMachineFileLayoutExFileInfo file : fileInfo) {
                  DatastoreFile fileInDatastore = new DatastoreFile(file.getName());
                  // In case of linked clones, VM file layout includes the base disk so don't delete all disk files.
                  if (file.getType().startsWith("disk") || file.getType().startsWith("digest"))
                      continue;
                  else if (file.getType().equals("config"))
                      vmFolder = new DatastoreFile(fileInDatastore.getDatastoreName(), fileInDatastore.getDir());
                  DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName()));
                  checkAndDeleteDatastoreFile(file.getName(), skipDatastores, dsMo, dcMo);
              }
              // Delete files that are present in the VM folder - this will take care of the VM disks as well.
              DatastoreMO vmFolderDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(vmFolder.getDatastoreName()));
              String[] files = vmFolderDsMo.listDirContent(vmFolder.getPath());
              if (deleteDisks) {
                  for (String file : files) {
                      String vmDiskFileFullPath = String.format("%s/%s", vmFolder.getPath(), file);
                      checkAndDeleteDatastoreFile(vmDiskFileFullPath, skipDatastores, vmFolderDsMo, dcMo);
                  }
              }
              // Delete VM folder
              if (deleteDisks || files.length == 0) {
                  checkAndDeleteDatastoreFile(vmFolder.getPath(), skipDatastores, vmFolderDsMo, dcMo);
              }
          } catch (Exception e) {
              String message = "Failed to delete files associated with an existing VM that was unregistered due to " + VmwareHelper.getExceptionMessage(e);
              logger.warn(message, e);
          }
      }
  
      private static VolumeObjectTO getVolumeInSpec(VirtualMachineTO vmSpec, VolumeObjectTO srcVol) {
          for (DiskTO disk : vmSpec.getDisks()) {
              if (disk.getData() instanceof VolumeObjectTO) {
                  VolumeObjectTO vol = (VolumeObjectTO) disk.getData();
                  if (vol.getId() == srcVol.getId())
                      return vol;
              }
          }
  
          return null;
      }
  
      private static NicTO[] sortNicsByDeviceId(NicTO[] nics) {
  
          List<NicTO> listForSort = new ArrayList<>();
          for (NicTO nic : nics) {
              listForSort.add(nic);
          }
          Collections.sort(listForSort, new Comparator<>() {
  
              @Override
              public int compare(NicTO arg0, NicTO arg1) {
                  if (arg0.getDeviceId() < arg1.getDeviceId()) {
                      return -1;
                  } else if (arg0.getDeviceId() == arg1.getDeviceId()) {
                      return 0;
                  }
  
                  return 1;
              }
          });
  
          return listForSort.toArray(new NicTO[0]);
      }
  
      private static DiskTO[] sortVolumesByDeviceId(DiskTO[] volumes) {
  
          List<DiskTO> listForSort = new ArrayList<>();
          for (DiskTO vol : volumes) {
              listForSort.add(vol);
          }
          Collections.sort(listForSort, new Comparator<>() {
  
              @Override
              public int compare(DiskTO arg0, DiskTO arg1) {
                  if (arg0.getDiskSeq() < arg1.getDiskSeq()) {
                      return -1;
                  } else if (arg0.getDiskSeq().equals(arg1.getDiskSeq())) {
                      return 0;
                  }
  
                  return 1;
              }
          });
  
          return listForSort.toArray(new DiskTO[0]);
      }
  
      /**
       * Only call this for managed storage.
       * Ex. "[-iqn.2010-01.com.solidfire:4nhe.vol-1.27-0] i-2-18-VM/ROOT-18.vmdk" should return "i-2-18-VM/ROOT-18"
       */
      public String getVmdkPath(String path) {
          if (StringUtils.isBlank(path)) {
              return null;
          }
  
          final String search = "]";
  
          int startIndex = path.indexOf(search);
  
          if (startIndex == -1) {
              return null;
          }
  
          path = path.substring(startIndex + search.length());
  
          int endIndex = path.indexOf(VMDK_EXTENSION);
  
          if (endIndex == -1) {
              return null;
          }
  
          return path.substring(0, endIndex).trim();
      }
  
      private DatastoreMO getDataStoreWhereDiskExists(VmwareHypervisorHost hyperHost, VmwareContext context,
                                                 VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO disk, List<Pair<Integer, ManagedObjectReference>> diskDatastores) throws Exception {
          VolumeObjectTO volume = (VolumeObjectTO) disk.getData();
          String diskBackingFileBaseName = volume.getPath();
          for (Pair<Integer, ManagedObjectReference> diskDatastore : diskDatastores) {
              DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), diskDatastore.second());
              String dsName = dsMo.getName();
  
              VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(diskBackingFileBaseName, dsName);
              if (diskInfo != null) {
                  logger.info("Found existing disk info from volume path: " + volume.getPath());
                  return dsMo;
              }
  
              String chainInfo = volume.getChainInfo();
              if (chainInfo != null) {
                  VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class);
                  if (infoInChain != null) {
                      String[] disks = infoInChain.getDiskChain();
                      if (disks.length > 0) {
                          for (String diskPath : disks) {
                              DatastoreFile file = new DatastoreFile(diskPath);
                              diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName);
                              if (diskInfo != null) {
                                  logger.info("Found existing disk from chain info: " + diskPath);
                                  return dsMo;
                              }
                          }
                      }
                  }
              }
          }
          return null;
      }
  
      private HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> inferDatastoreDetailsFromDiskInfo(VmwareHypervisorHost hyperHost, VmwareContext context,
                                                                                                           DiskTO[] disks, Command cmd) throws Exception {
          HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> mapIdToMors = new HashMap<>();
  
          assert (hyperHost != null) && (context != null);
  
          for (DiskTO vol : disks) {
              if (vol.getType() != Volume.Type.ISO) {
                  VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
                  DataStoreTO primaryStore = volumeTO.getDataStore();
                  String poolUuid = primaryStore.getUuid();
  
                  if (mapIdToMors.get(poolUuid) == null) {
                      boolean isManaged = false;
                      Map<String, String> details = vol.getDetails();
  
                      if (details != null) {
                          isManaged = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
                      }
  
                      if (isManaged) {
                          String iScsiName = details.get(DiskTO.IQN); // details should not be null for managed storage (it may or may not be null for non-managed storage)
                          String datastoreName = VmwareResource.getDatastoreName(iScsiName);
                          ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, datastoreName);
  
                          // if the datastore is not present, we need to discover the iSCSI device that will support it,
                          // create the datastore, and create a VMDK file in the datastore
                          if (morDatastore == null) {
                              final String vmdkPath = getVmdkPath(volumeTO.getPath());
  
                              morDatastore = _storageProcessor.prepareManagedStorage(context, hyperHost, null, iScsiName,
                                      details.get(DiskTO.STORAGE_HOST), Integer.parseInt(details.get(DiskTO.STORAGE_PORT)),
                                      vmdkPath,
                                      details.get(DiskTO.CHAP_INITIATOR_USERNAME), details.get(DiskTO.CHAP_INITIATOR_SECRET),
                                      details.get(DiskTO.CHAP_TARGET_USERNAME), details.get(DiskTO.CHAP_TARGET_SECRET),
                                      Long.parseLong(details.get(DiskTO.VOLUME_SIZE)), cmd);
  
                              DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore);
  
                              final String datastoreVolumePath;
  
                              if (vmdkPath != null) {
                                  datastoreVolumePath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION);
                              } else {
                                  datastoreVolumePath = dsMo.getDatastorePath(dsMo.getName() + VMDK_EXTENSION);
                              }
  
                              volumeTO.setPath(datastoreVolumePath);
                              vol.setPath(datastoreVolumePath);
                          }
  
                          mapIdToMors.put(datastoreName, new Pair<>(morDatastore, new DatastoreMO(context, morDatastore)));
                      } else {
                          ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolUuid);
  
                          if (morDatastore == null) {
                              String msg = "Failed to get the mounted datastore for the volume's pool " + poolUuid;
  
                              logger.error(msg);
  
                              throw new Exception(msg);
                          }
  
                          mapIdToMors.put(poolUuid, new Pair<>(morDatastore, new DatastoreMO(context, morDatastore)));
                      }
                  }
              }
          }
  
          return mapIdToMors;
      }
  
      private Pair<ManagedObjectReference, DatastoreMO> getDatastoreThatDiskIsOn(HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> dataStoresDetails, DiskTO vol) {
          Pair<ManagedObjectReference, DatastoreMO> rootDiskDataStoreDetails;
  
          Map<String, String> details = vol.getDetails();
          boolean managed = false;
  
          if (details != null) {
              managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
          }
  
          if (managed) {
              String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN));
              rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName);
          } else {
              DataStoreTO primaryStore = vol.getData().getDataStore();
              rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid());
          }
  
          return rootDiskDataStoreDetails;
      }
  
      private String getPvlanInfo(NicTO nicTo) {
          if (nicTo.getBroadcastType() == BroadcastDomainType.Pvlan) {
              return NetUtils.getIsolatedPvlanFromUri(nicTo.getBroadcastUri());
          }
          return null;
      }
  
      private String getVlanInfo(NicTO nicTo, String defaultVlan) {
          if (nicTo.getBroadcastType() == BroadcastDomainType.Native) {
              return defaultVlan;
          } else if (nicTo.getBroadcastType() == BroadcastDomainType.Vlan || nicTo.getBroadcastType() == BroadcastDomainType.Pvlan) {
              if (nicTo.getBroadcastUri() != null) {
                  if (nicTo.getBroadcastType() == BroadcastDomainType.Vlan)
                      // For vlan, the broadcast uri is of the form vlan://<vlanid>
                      // BroadcastDomainType recogniizes and handles this.
                      return BroadcastDomainType.getValue(nicTo.getBroadcastUri());
                  else
                      // for pvlan, the broadcast uri will be of the form pvlan://<vlanid>-i<pvlanid>
                      // TODO consider the spread of functionality between BroadcastDomainType and NetUtils
                      return NetUtils.getPrimaryPvlanFromUri(nicTo.getBroadcastUri());
              } else {
                  logger.warn("BroadcastType is not claimed as VLAN or PVLAN, but without vlan info in broadcast URI. Use vlan info from labeling: " + defaultVlan);
                  return defaultVlan;
              }
          } else if (nicTo.getBroadcastType() == BroadcastDomainType.Lswitch) {
              // We don't need to set any VLAN id for an NVP logical switch
              return null;
          } else if (nicTo.getBroadcastType() == BroadcastDomainType.Storage) {
              URI broadcastUri = nicTo.getBroadcastUri();
              if (broadcastUri != null) {
                  String vlanId = BroadcastDomainType.getValue(broadcastUri);
                  logger.debug("Using VLAN [" + vlanId + "] from broadcast uri [" + broadcastUri + "]");
                  return vlanId;
              }
          }
  
          logger.warn("Unrecognized broadcast type in VmwareResource, type: " + nicTo.getBroadcastType().toString() + ". Use vlan info from labeling: " + defaultVlan);
          return defaultVlan;
      }
  
      private Pair<ManagedObjectReference, String> prepareNetworkFromNicInfo(HostMO hostMo, NicTO nicTo, boolean configureVServiceInNexus, String networkName, VirtualMachine.Type vmType)
              throws Exception {
  
          Ternary<String, String, String> switchDetails = getTargetSwitch(nicTo);
          VirtualSwitchType switchType = VirtualSwitchType.getType(switchDetails.second());
          String switchName = switchDetails.first();
          String vlanToken = switchDetails.third();
  
          String namePrefix = getNetworkNamePrefix(nicTo);
          Pair<ManagedObjectReference, String> networkInfo;
  
          logger.info("Prepare network on " + switchType + " " + switchName + " with name prefix: " + namePrefix);
  
          if (VirtualSwitchType.StandardVirtualSwitch == switchType) {
              networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo,
                      getVlanInfo(nicTo, vlanToken), nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(),
                      _opsTimeout, true, nicTo.getBroadcastType(), nicTo.getUuid(), nicTo.getDetails());
          } else {
              String vlanId = getVlanInfo(nicTo, vlanToken);
              String svlanId;
              boolean pvlannetwork = (getPvlanInfo(nicTo) == null) ? false : true;
              if (vmType != null && vmType.equals(VirtualMachine.Type.DomainRouter) && pvlannetwork) {
                  // plumb this network to the promiscuous vlan.
                  svlanId = vlanId;
              } else {
                  // plumb this network to the isolated vlan.
                  svlanId = getPvlanInfo(nicTo);
              }
              networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo, vlanId, svlanId,
                      nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _opsTimeout, switchType,
                      _portsPerDvPortGroup, nicTo.getGateway(), configureVServiceInNexus, nicTo.getBroadcastType(), _vsmCredentials, nicTo.getDetails(), networkName);
          }
  
          return networkInfo;
      }
  
      // return Ternary <switch name, switch type, vlan tagging>
      private Ternary<String, String, String> getTargetSwitch(NicTO nicTo) throws CloudException {
          TrafficType[] supportedTrafficTypes = new TrafficType[]{TrafficType.Guest, TrafficType.Public, TrafficType.Control, TrafficType.Management, TrafficType.Storage};
  
          TrafficType trafficType = nicTo.getType();
          if (!Arrays.asList(supportedTrafficTypes).contains(trafficType)) {
              throw new CloudException("Traffic type " + trafficType.toString() + " for nic " + nicTo + " is not supported.");
          }
  
          String switchName = null;
          VirtualSwitchType switchType = VirtualSwitchType.StandardVirtualSwitch;
          String vlanId = Vlan.UNTAGGED;
  
          if (StringUtils.isNotBlank(nicTo.getName())) {
              // Format of network traffic label is <VSWITCH>,<VLANID>,<VSWITCHTYPE>
              // If all 3 fields are mentioned then number of tokens would be 3.
              // If only <VSWITCH>,<VLANID> are mentioned then number of tokens would be 2.
              // Get switch details from the nicTO object
              String networkName = nicTo.getName();
              VmwareTrafficLabel mgmtTrafficLabelObj = new VmwareTrafficLabel(networkName, trafficType);
              switchName = mgmtTrafficLabelObj.getVirtualSwitchName();
              vlanId = mgmtTrafficLabelObj.getVlanId();
              switchType = mgmtTrafficLabelObj.getVirtualSwitchType();
          } else {
              if (trafficType == TrafficType.Guest && _guestTrafficInfo != null) {
                  switchType = _guestTrafficInfo.getVirtualSwitchType();
                  switchName = _guestTrafficInfo.getVirtualSwitchName();
              } else if (trafficType == TrafficType.Public && _publicTrafficInfo != null) {
                  switchType = _publicTrafficInfo.getVirtualSwitchType();
                  switchName = _publicTrafficInfo.getVirtualSwitchName();
              }
          }
  
          if (switchName == null
                  && (nicTo.getType() == Networks.TrafficType.Control || nicTo.getType() == Networks.TrafficType.Management || nicTo.getType() == Networks.TrafficType.Storage)) {
              switchName = _privateNetworkVSwitchName;
          }
  
          if (switchType == VirtualSwitchType.NexusDistributedVirtualSwitch) {
              if (trafficType == TrafficType.Management || trafficType == TrafficType.Storage) {
                  throw new CloudException(
                          "Unable to configure NIC " + nicTo + " as traffic type " + trafficType + " is not supported over virtual switch type " + switchType
                                  + ". Please specify only supported type of virtual switches i.e. {vmwaresvs, vmwaredvs} in physical network traffic label.");
              }
          }
  
          return new Ternary<>(switchName, switchType.toString(), vlanId);
      }
  
      private String getNetworkNamePrefix(NicTO nicTo) throws Exception {
          if (nicTo.getType() == Networks.TrafficType.Guest) {
              return "cloud.guest";
          } else if (nicTo.getType() == Networks.TrafficType.Control || nicTo.getType() == Networks.TrafficType.Management) {
              return "cloud.private";
          } else if (nicTo.getType() == Networks.TrafficType.Public) {
              return "cloud.public";
          } else if (nicTo.getType() == Networks.TrafficType.Storage) {
              return "cloud.storage";
          } else if (nicTo.getType() == Networks.TrafficType.Vpn) {
              throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
          } else {
              throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
          }
      }
  
      private String getControlIp(NicTO[] nicTOs) {
          String controlIpAddress = null;
          for (NicTO nic : nicTOs) {
              if ((TrafficType.Management == nic.getType() || TrafficType.Control == nic.getType()) && nic.getIp() != null) {
                  controlIpAddress = nic.getIp();
                  break;
              }
          }
          return controlIpAddress;
      }
  
      private VirtualMachineMO takeVmFromOtherHyperHost(VmwareHypervisorHost hyperHost, String vmName) throws Exception {
  
          VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
          if (vmMo != null) {
              ManagedObjectReference morTargetPhysicalHost = hyperHost.findMigrationTarget(vmMo);
              if (morTargetPhysicalHost == null) {
                  String msg = "VM " + vmName + " is on other host and we have no resource available to migrate and start it here";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              if (!vmMo.relocate(morTargetPhysicalHost)) {
                  String msg = "VM " + vmName + " is on other host and we failed to relocate it here";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              return vmMo;
          }
          return null;
      }
  
      // isoUrl sample content :
      // nfs://192.168.10.231/export/home/kelven/vmware-test/secondary/template/tmpl/2/200//200-2-80f7ee58-6eff-3a2d-bcb0-59663edf6d26.iso
      private Pair<String, ManagedObjectReference> getIsoDatastoreInfo(VmwareHypervisorHost hyperHost, String isoUrl) throws Exception {
  
          assert (isoUrl != null);
          int isoFileNameStartPos = isoUrl.lastIndexOf("/");
          if (isoFileNameStartPos < 0) {
              throw new Exception("Invalid ISO path info");
          }
  
          String isoFileName = isoUrl.substring(isoFileNameStartPos);
  
          int templateRootPos = isoUrl.indexOf("template/tmpl");
          templateRootPos = (templateRootPos < 0 ? isoUrl.indexOf(ConfigDrive.CONFIGDRIVEDIR) : templateRootPos);
          if (templateRootPos < 0) {
              throw new Exception("Invalid ISO path info");
          }
  
          String storeUrl = isoUrl.substring(0, templateRootPos - 1);
          String isoPath = isoUrl.substring(templateRootPos, isoFileNameStartPos);
  
          ManagedObjectReference morDs = prepareSecondaryDatastoreOnHost(storeUrl);
          DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDs);
  
          return new Pair<>(String.format("[%s] %s%s", dsMo.getName(), isoPath, isoFileName), morDs);
      }
  
      protected Answer execute(ReadyCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              Map<String, String> hostDetails = new HashMap<>();
              ManagedObjectReference morHost = hyperHost.getMor();
              HostMO hostMo = new HostMO(context, morHost);
              boolean uefiLegacySupported = hostMo.isUefiLegacySupported();
              if (uefiLegacySupported) {
                  hostDetails.put(Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString());
              }
  
              if (hyperHost.isHyperHostConnected()) {
                  return new ReadyAnswer(cmd, hostDetails);
              } else {
                  return new ReadyAnswer(cmd, "Host is not in connect state");
              }
          } catch (Exception e) {
              logger.error("Unexpected exception: ", e);
              return new ReadyAnswer(cmd, VmwareHelper.getExceptionMessage(e));
          }
      }
  
      protected Answer execute(GetHostStatsCommand cmd) {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          HostStatsEntry hostStats = new HostStatsEntry(cmd.getHostId(), 0, 0, 0, "host", 0, 0, 0, 0);
          Answer answer = new GetHostStatsAnswer(cmd, hostStats);
          try {
              HostStatsEntry entry = getHyperHostStats(hyperHost);
              if (entry != null) {
                  logger.debug(String.format("Host stats response from hypervisor is: [%s].", _gson.toJson(entry)));
                  entry.setHostId(cmd.getHostId());
                  answer = new GetHostStatsAnswer(cmd, entry);
              }
          } catch (Exception e) {
              logger.error(createLogMessageException(e, cmd), e);
          }
  
          if (logger.isTraceEnabled()) {
              logger.trace("GetHostStats Answer: " + _gson.toJson(answer));
          }
  
          return answer;
      }
  
      protected Answer execute(GetVmStatsCommand cmd) {
          HashMap<String, VmStatsEntry> vmStatsMap = null;
  
          try {
              HashMap<String, PowerState> vmPowerStates = getVmStates();
  
              // getVmNames should return all i-x-y values.
              List<String> requestedVmNames = cmd.getVmNames();
              List<String> vmNames = new ArrayList<>();
  
              if (requestedVmNames != null) {
                  for (String vmName : requestedVmNames) {
                      if (vmPowerStates.get(vmName) != null) {
                          vmNames.add(vmName);
                      }
                  }
              }
  
              if (CollectionUtils.isNotEmpty(vmNames)) {
                  vmStatsMap = getVmStats(vmNames);
              }
          } catch (Throwable e) {
              createLogMessageException(e, cmd);
          }
  
          logger.debug(String.format("VM Stats Map is: [%s].", _gson.toJson(vmStatsMap)));
          return new GetVmStatsAnswer(cmd, vmStatsMap);
      }
  
      protected Answer execute(GetVmDiskStatsCommand cmd) {
          try {
              final VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              final ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager();
              VimPortType service = getServiceContext().getService();
  
              Integer windowInterval = getVmwareWindowTimeInterval();
              final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), windowInterval);
              final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0);
  
              PerfCounterInfo diskReadIOPerfCounterInfo = null;
              PerfCounterInfo diskWriteIOPerfCounterInfo = null;
              PerfCounterInfo diskReadKbsPerfCounterInfo = null;
              PerfCounterInfo diskWriteKbsPerfCounterInfo = null;
  
              // https://pubs.vmware.com/vsphere-5-5/topic/com.vmware.wssdk.apiref.doc/virtual_disk_counters.html
              List<PerfCounterInfo> cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter");
              for (PerfCounterInfo info : cInfo) {
                  if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) {
                      if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
                          diskReadIOPerfCounterInfo = info;
                      }
                      if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
                          diskWriteIOPerfCounterInfo = info;
                      }
                      if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) {
                          diskReadKbsPerfCounterInfo = info;
                      }
                      if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) {
                          diskWriteKbsPerfCounterInfo = info;
                      }
                  }
              }
  
              final ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
              final DatacenterMO dcMo = new DatacenterMO(getServiceContext(), dcMor);
  
              final HashMap<String, List<VmDiskStatsEntry>> vmStatsMap = new HashMap<>();
              for (final String vmName : cmd.getVmNames()) {
                  final VirtualMachineMO vmMo = dcMo.findVm(vmName);
                  final List<VmDiskStatsEntry> diskStats = new ArrayList<>();
                  for (final VirtualDisk disk : vmMo.getAllDiskDevice()) {
                      final String diskBusName = vmMo.getDeviceBusName(vmMo.getAllDeviceList(), disk);
                      long readReq = 0;
                      long readBytes = 0;
                      long writeReq = 0;
                      long writeBytes = 0;
  
                      final ArrayList<PerfMetricId> perfMetricsIds = new ArrayList<>();
                      if (diskReadIOPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, diskBusName));
                      }
                      if (diskWriteIOPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, diskBusName));
                      }
                      if (diskReadKbsPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, diskBusName));
                      }
                      if (diskWriteKbsPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, diskBusName));
                      }
  
                      if (perfMetricsIds.size() > 0) {
                          try {
                              final PerfQuerySpec qSpec = new PerfQuerySpec();
                              qSpec.setEntity(vmMo.getMor());
                              qSpec.setFormat("normal");
                              qSpec.setIntervalId(windowInterval);
                              qSpec.setStartTime(startTime);
                              qSpec.setEndTime(endTime);
                              qSpec.getMetricId().addAll(perfMetricsIds);
  
                              for (final PerfEntityMetricBase perfValue: service.queryPerf(perfMgr, Collections.singletonList(qSpec))) {
                                  if (!(perfValue instanceof PerfEntityMetric)) {
                                      continue;
                                  }
                                  final List<PerfMetricSeries> values = ((PerfEntityMetric) perfValue).getValue();
                                  if (values == null || values.isEmpty()) {
                                      continue;
                                  }
                                  for (final PerfMetricSeries value : values) {
                                      if (!(value instanceof PerfMetricIntSeries) || !value.getId().getInstance().equals(diskBusName)) {
                                          continue;
                                      }
                                      final List<Long> perfStats = ((PerfMetricIntSeries) value).getValue();
                                      if (perfStats.size() > 0) {
                                          long sum = 0;
                                          for (long val : perfStats) {
                                              sum += val;
                                          }
                                          long avg = sum / perfStats.size();
                                          if (value.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) {
                                              readReq = avg;
                                          } else if (value.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) {
                                              writeReq = avg;
                                          } else if (value.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) {
                                              readBytes = avg * 1024;
                                          } else if (value.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) {
                                              writeBytes = avg * 1024;
                                          }
                                      }
                                  }
                              }
                          } catch (Exception e) {
                              logger.error(String.format("Unable to execute PerfQuerySpec due to: [%s]. The window interval is enabled in vCenter?", VmwareHelper.getExceptionMessage(e)), e);
                          }
  
                      }
                      diskStats.add(new VmDiskStatsEntry(vmName, VmwareHelper.getDiskDeviceFileName(disk), writeReq, readReq, writeBytes, readBytes));
                  }
                  if (CollectionUtils.isNotEmpty(diskStats)) {
                      vmStatsMap.put(vmName, diskStats);
                  }
              }
  
              logger.debug(String.format("VM Disks Maps is: [%s].", _gson.toJson(vmStatsMap)));
              if (MapUtils.isNotEmpty(vmStatsMap)) {
                  return new GetVmDiskStatsAnswer(cmd, "", cmd.getHostName(), vmStatsMap);
              }
          } catch (Exception e) {
              logger.error(String.format("Unable to execute GetVmDiskStatsCommand due to [%s].", VmwareHelper.getExceptionMessage(e)), e);
          }
          return new GetVmDiskStatsAnswer(cmd, null, null, null);
      }
  
      protected Answer execute(GetVmNetworkStatsCommand cmd) {
          return new GetVmNetworkStatsAnswer(cmd, null, null, null);
      }
  
      protected GetVolumeStatsAnswer execute(GetVolumeStatsCommand cmd) {
          try {
              VmwareHypervisorHost srcHyperHost = getHyperHost(getServiceContext());
              ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, cmd.getPoolUuid());
              assert (morDs != null);
              DatastoreMO primaryStorageDatastoreMo = new DatastoreMO(getServiceContext(), morDs);
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
              DatacenterMO dcMo = new DatacenterMO(getServiceContext(), dcMor);
              HashMap<String, VolumeStatsEntry> statEntry = new HashMap<>();
  
              for (String chainInfo : cmd.getVolumeUuids()) {
                  if (chainInfo != null) {
                      VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class);
                      if (infoInChain != null) {
                          String[] disks = infoInChain.getDiskChain();
                          if (disks.length > 0) {
                              for (String diskPath : disks) {
                                  DatastoreFile file = new DatastoreFile(diskPath);
                                  VirtualMachineMO vmMo = dcMo.findVm(file.getDir());
                                  Pair<VirtualDisk, String> vds = vmMo.getDiskDevice(file.getFileName(), true);
                                  long virtualsize = vds.first().getCapacityInKB() * 1024;
                                  long physicalsize = primaryStorageDatastoreMo.fileDiskSize(file.getPath());
                                  if (statEntry.containsKey(chainInfo)) {
                                      VolumeStatsEntry vse = statEntry.get(chainInfo);
                                      if (vse != null) {
                                          vse.setPhysicalSize(vse.getPhysicalSize() + physicalsize);
                                      }
                                  } else {
                                      VolumeStatsEntry vse = new VolumeStatsEntry(chainInfo, physicalsize, virtualsize);
                                      statEntry.put(chainInfo, vse);
                                  }
                              }
                          }
                      }
                  }
              }
              logger.debug(String.format("Volume Stats Entry is: [%s].", _gson.toJson(statEntry)));
              return new GetVolumeStatsAnswer(cmd, "", statEntry);
          } catch (Exception e) {
              logger.error(String.format("VOLSTAT GetVolumeStatsCommand failed due to [%s].", VmwareHelper.getExceptionMessage(e)), e);
          }
  
          return new GetVolumeStatsAnswer(cmd, "", null);
      }
  
      protected Answer execute(CheckHealthCommand cmd) {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              if (hyperHost.isHyperHostConnected()) {
                  return new CheckHealthAnswer(cmd, true);
              }
          } catch (Throwable e) {
              createLogMessageException(e, cmd);
          }
          return new CheckHealthAnswer(cmd, false);
      }
  
      protected Answer execute(StopCommand cmd) {
          // In the stop command, we're passed in the name of the VM as seen by cloudstack,
          // i.e., i-x-y. This is the internal VM name.
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          try {
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              if (vmMo != null) {
                  if (cmd.checkBeforeCleanup()) {
                      if (getVmPowerState(vmMo) != PowerState.PowerOff) {
                          String msg = "StopCommand is sent for cleanup and VM " + cmd.getVmName() + " is current running. ignore it.";
                          logger.warn(msg);
                          return new StopAnswer(cmd, msg, false);
                      } else {
                          String msg = "StopCommand is sent for cleanup and VM " + cmd.getVmName() + " is indeed stopped already.";
                          logger.info(msg);
                          return new StopAnswer(cmd, msg, true);
                      }
                  }
  
                  try {
                      vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, "0");
                      vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_VM_INTERNAL_NAME, cmd.getVmName());
  
                      if (getVmPowerState(vmMo) != PowerState.PowerOff) {
                          String msg = "Stop VM " + cmd.getVmName() + " Succeed";
                          boolean success;
                          if (cmd.isForceStop()) {
                              success = vmMo.powerOff();
                          } else {
                              success = vmMo.safePowerOff(_shutdownWaitMs);
                          }
                          if (!success) {
                              msg = "Have problem in powering off VM " + cmd.getVmName() + ", let the process continue";
                              logger.warn(msg);
                          }
  
                          disconnectConfigDriveIsoIfExists(vmMo);
                          return new StopAnswer(cmd, msg, true);
                      }
  
                      String msg = "VM " + cmd.getVmName() + " is already in stopped state";
                      logger.info(msg);
                      return new StopAnswer(cmd, msg, true);
                  } finally {
                  }
              } else {
                  String msg = "VM " + cmd.getVmName() + " is no longer on the expected host in vSphere";
                  logger.info(msg);
                  return new StopAnswer(cmd, msg, true);
              }
          } catch (Exception e) {
              return new StopAnswer(cmd, createLogMessageException(e, cmd), false);
          }
      }
  
      private void disconnectConfigDriveIsoIfExists(VirtualMachineMO vmMo) {
          try {
              List<VirtualDevice> isoDevices = vmMo.getIsoDevices();
              if (CollectionUtils.isEmpty(isoDevices)) {
                  return;
              }
  
              for (VirtualDevice isoDevice : isoDevices) {
                  if (!(isoDevice.getBacking() instanceof VirtualCdromIsoBackingInfo)) {
                      continue;
                  }
                  String isoFilePath = ((VirtualCdromIsoBackingInfo)isoDevice.getBacking()).getFileName();
                  if (!isoFilePath.contains(ConfigDrive.CONFIGDRIVEDIR)) {
                      continue;
                  }
                  logger.info(String.format("Disconnecting config drive at location: %s", isoFilePath));
                  vmMo.detachIso(isoFilePath, true);
                  return;
              }
          } catch (Exception e) {
              logger.warn(String.format("Couldn't check/disconnect config drive, error: %s", e.getMessage()), e);
          }
      }
  
      protected Answer execute(RebootRouterCommand cmd) {
          RebootAnswer answer = (RebootAnswer) execute((RebootCommand) cmd);
  
          if (answer.getResult()) {
              String connectResult = connect(cmd.getVmName(), cmd.getPrivateIpAddress());
              networkUsage(cmd.getPrivateIpAddress(), "create", null);
              if (connectResult == null) {
                  return answer;
              } else {
                  return new Answer(cmd, false, connectResult);
              }
          }
          return answer;
      }
  
      protected Answer execute(RebootCommand cmd) {
          boolean toolsInstallerMounted = false;
          VirtualMachineMO vmMo = null;
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          try {
              vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              if (vmMo != null) {
                  if (vmMo.isToolsInstallerMounted()) {
                      toolsInstallerMounted = true;
                      logger.trace("Detected mounted vmware tools installer for :[" + cmd.getVmName() + "]");
                  }
                  try {
                      if (canSetEnableSetupConfig(vmMo,cmd.getVirtualMachine())) {
                          vmMo.rebootGuest();
                          return new RebootAnswer(cmd, "reboot succeeded", true);
                      } else {
                          return new RebootAnswer(cmd, "Failed to configure VM to boot into hardware setup menu: " + vmMo.getName(), false);
                      }
                  } catch (ToolsUnavailableFaultMsg e) {
                      logger.warn("VMware tools is not installed at guest OS, we will perform hard reset for reboot");
                  } catch (Exception e) {
                      logger.warn("We are not able to perform gracefull guest reboot due to " + VmwareHelper.getExceptionMessage(e));
                  }
  
                  // continue to try with hard-reset
                  if (vmMo.reset()) {
                      return new RebootAnswer(cmd, "reboot succeeded", true);
                  }
  
                  String msg = "Reboot failed in vSphere. vm: " + cmd.getVmName();
                  logger.warn(msg);
                  return new RebootAnswer(cmd, msg, false);
              } else {
                  String msg = "Unable to find the VM in vSphere to reboot. vm: " + cmd.getVmName();
                  logger.warn(msg);
                  return new RebootAnswer(cmd, msg, false);
              }
          } catch (Exception e) {
              return new RebootAnswer(cmd, createLogMessageException(e, cmd), false);
          } finally {
              if (toolsInstallerMounted) {
                  try {
                      vmMo.mountToolsInstaller();
                      logger.debug(String.format("Successfully re-mounted vmware tools installer for :[%s].", cmd.getVmName()));
                  } catch (Exception e) {
                      logger.error(String.format("Unabled to re-mount vmware tools installer for: [%s].", cmd.getVmName()), e);
                  }
              }
          }
      }
  
      /**
       * set the boot into setup option if possible
       * @param vmMo vmware view on the vm
       * @param virtualMachine orchestration spec for the vm
       * @return true unless reboot into setup is requested and vmware is unable to comply
       */
      private boolean canSetEnableSetupConfig(VirtualMachineMO vmMo, VirtualMachineTO virtualMachine) {
          if (virtualMachine.isEnterHardwareSetup()) {
              VirtualMachineBootOptions bootOptions = new VirtualMachineBootOptions();
              VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
              if (logger.isDebugEnabled()) {
                  logger.debug(String.format("configuring VM '%s' to reboot into hardware setup menu.",virtualMachine.getName()));
              }
              bootOptions.setEnterBIOSSetup(virtualMachine.isEnterHardwareSetup());
              vmConfigSpec.setBootOptions(bootOptions);
              try {
                  if (!vmMo.configureVm(vmConfigSpec)) {
                      return false;
                  }
              } catch (Exception e) {
                  logger.error(String.format("failed to reconfigure VM '%s' to boot into hardware setup menu",virtualMachine.getName()),e);
                  return false;
              }
          }
          return true;
      }
  
      protected Answer execute(CheckVirtualMachineCommand cmd) {
          final String vmName = cmd.getVmName();
          PowerState powerState = PowerState.PowerUnknown;
          Integer vncPort = null;
  
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          try {
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
              if (vmMo != null) {
                  powerState = getVmPowerState(vmMo);
                  return new CheckVirtualMachineAnswer(cmd, powerState, vncPort);
              } else {
                  logger.warn("Can not find vm " + vmName + " to execute CheckVirtualMachineCommand");
                  return new CheckVirtualMachineAnswer(cmd, powerState, vncPort);
              }
  
          } catch (Throwable e) {
              createLogMessageException(e, cmd);
              return new CheckVirtualMachineAnswer(cmd, powerState, vncPort);
          }
      }
  
      protected Answer execute(PrepareForMigrationCommand cmd) {
          VirtualMachineTO vm = cmd.getVirtualMachine();
          final String vmName = vm.getName();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              // find VM through datacenter (VM is not at the target host yet)
              VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              if (vmMo == null) {
                  logger.info("VM " + vmName + " was not found in the cluster of host " + hyperHost.getHyperHostName() + ". Looking for the VM in datacenter.");
                  ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
                  DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), dcMor);
                  vmMo = dcMo.findVm(vmName);
                  if (vmMo == null) {
                      String msg = "VM " + vmName + " does not exist in VMware datacenter";
                      logger.error(msg);
                      throw new Exception(msg);
                  }
              }
  
              NicTO[] nics = vm.getNics();
              for (NicTO nic : nics) {
                  // prepare network on the host
                  prepareNetworkFromNicInfo(new HostMO(getServiceContext(), _morHyperHost), nic, false,
                          vm.getNetworkIdToNetworkNameMap().getOrDefault(nic.getNetworkId(), null), cmd.getVirtualMachine().getType());
              }
  
              List<Pair<String, Long>> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId));
              for (Pair<String, Long> secStoreUrlAndId : secStoreUrlAndIdList) {
                  String secStoreUrl = secStoreUrlAndId.first();
                  if (secStoreUrl == null) {
                      String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used",
                              _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100));
                      throw new Exception(msg);
                  }
  
                  ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl);
                  if (morSecDs == null) {
                      String msg = "Failed to prepare secondary storage on host, NFS secondary or cache store url: " + secStoreUrl + " in dc "+ _dcId;
                      throw new Exception(msg);
                  }
              }
              return new PrepareForMigrationAnswer(cmd);
          } catch (Throwable e) {
              return new PrepareForMigrationAnswer(cmd, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(MigrateVmToPoolCommand cmd) {
          final String vmName = cmd.getVmName();
  
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          try {
              VirtualMachineMO vmMo = getVirtualMachineMO(vmName, hyperHost);
              if (vmMo == null) {
                  logger.info("VM " + vmName + " was not found in the cluster of host " + hyperHost.getHyperHostName() + ". Looking for the VM in datacenter.");
                  ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
                  DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), dcMor);
                  vmMo = dcMo.findVm(vmName);
                  if (vmMo == null) {
                      String msg = "VM " + vmName + " does not exist in VMware datacenter";
                      logger.error(msg);
                      throw new CloudRuntimeException(msg);
                  }
              }
              return migrateAndAnswer(vmMo, null, hyperHost, cmd);
          } catch (Throwable e) { // hopefully only CloudRuntimeException :/
              if (e instanceof Exception) {
                  return new Answer(cmd, (Exception) e);
              }
              if (logger.isDebugEnabled()) {
                  logger.debug("problem", e);
              }
              logger.error(e.getLocalizedMessage());
              return new Answer(cmd, false, "unknown problem: " + e.getLocalizedMessage());
          }
      }
  
      private Answer migrateAndAnswer(VirtualMachineMO vmMo, String poolUuid, VmwareHypervisorHost hyperHost, Command cmd) throws Exception {
          String hostNameInTargetCluster = null;
          List<Pair<VolumeTO, StorageFilerTO>> volToFiler = new ArrayList<>();
          if (cmd instanceof MigrateVmToPoolCommand) {
              MigrateVmToPoolCommand mcmd = (MigrateVmToPoolCommand)cmd;
              hostNameInTargetCluster = mcmd.getHostGuidInTargetCluster();
              volToFiler = mcmd.getVolumeToFilerAsList();
          } else if (cmd instanceof MigrateVolumeCommand) {
              hostNameInTargetCluster = ((MigrateVolumeCommand)cmd).getHostGuidInTargetCluster();
          }
          VmwareHypervisorHost hostInTargetCluster = VmwareHelper.getHostMOFromHostName(getServiceContext(),
                  hostNameInTargetCluster);
  
          try {
              // OfflineVmwareMigration: getVolumesFromCommand(cmd);
              Map<Integer, Long> volumeDeviceKey = new HashMap<>();
              if (cmd instanceof MigrateVolumeCommand) { // Else device keys will be found in relocateVirtualMachine
                  MigrateVolumeCommand mcmd = (MigrateVolumeCommand) cmd;
                  addVolumeDiskMapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId());
                  if (logger.isTraceEnabled()) {
                      for (Integer diskId: volumeDeviceKey.keySet()) {
                          logger.trace(String.format("Disk to migrate has disk id %d and volumeId %d", diskId, volumeDeviceKey.get(diskId)));
                      }
                  }
              }
              List<VolumeObjectTO> volumeToList = relocateVirtualMachine(hyperHost, vmMo.getName(), null, null, hostInTargetCluster, poolUuid, volToFiler);
              return createAnswerForCmd(vmMo, volumeToList, cmd, volumeDeviceKey);
          } catch (Exception e) {
              String msg = "Change data store for VM " + vmMo.getVmName() + " failed";
              logger.error(msg + ": " + e.getLocalizedMessage());
              throw new CloudRuntimeException(msg, e);
          }
      }
  
      Answer createAnswerForCmd(VirtualMachineMO vmMo, List<VolumeObjectTO> volumeObjectToList, Command cmd, Map<Integer, Long> volumeDeviceKey) throws Exception {
          List<VolumeObjectTO> volumeToList;
          VirtualDisk[] disks = vmMo.getAllDiskDevice();
          if (logger.isTraceEnabled()) {
              logger.trace(String.format("creating answer for %s", cmd.getClass().getSimpleName()));
          }
          if (cmd instanceof MigrateVolumeCommand) {
              if (disks.length == 1) {
                  String volumePath = vmMo.getVmdkFileBaseName(disks[0]);
                  return new MigrateVolumeAnswer(cmd, true, null, volumePath);
              }
              throw new CloudRuntimeException("not expecting more then  one disk after migrate volume command");
          } else if (cmd instanceof MigrateVmToPoolCommand) {
              volumeToList = volumeObjectToList;
              return new MigrateVmToPoolAnswer((MigrateVmToPoolCommand)cmd, volumeToList);
          }
          return new Answer(cmd, false, null);
      }
  
      private void addVolumeDiskMapping(VirtualMachineMO vmMo, Map<Integer, Long> volumeDeviceKey, String volumePath, long volumeId) throws Exception {
          if (logger.isDebugEnabled()) {
              logger.debug(String.format("locating disk for volume (%d) using path %s", volumeId, volumePath));
          }
          Pair<VirtualDisk, String> diskInfo = getVirtualDiskInfo(vmMo, volumePath + VMDK_EXTENSION);
          String vmdkAbsFile = VmwareHelper.getAbsoluteVmdkFile(diskInfo.first());
          if (vmdkAbsFile != null && !vmdkAbsFile.isEmpty()) {
              vmMo.updateAdapterTypeIfRequired(vmdkAbsFile);
          }
          int diskId = diskInfo.first().getKey();
          volumeDeviceKey.put(diskId, volumeId);
      }
  
      private ManagedObjectReference getTargetDatastoreMOReference(String destinationPool,
                                                                   VmwareHypervisorHost hyperHost) {
          ManagedObjectReference morDs;
          try {
              if (logger.isDebugEnabled()) {
                  logger.debug(String.format("finding datastore %s", destinationPool));
              }
              morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, destinationPool);
          } catch (Exception e) {
              String msg = "exception while finding data store  " + destinationPool;
              logger.error(msg);
              throw new CloudRuntimeException(msg + ": " + e.getLocalizedMessage());
          }
          return morDs;
      }
  
      private ManagedObjectReference getDataCenterMOReference(String vmName, VmwareHypervisorHost hyperHost) {
          ManagedObjectReference morDc;
          try {
              morDc = hyperHost.getHyperHostDatacenter();
          } catch (Exception e) {
              String msg = "exception while finding VMware datacenter to search for VM " + vmName;
              logger.error(msg);
              throw new CloudRuntimeException(msg + ": " + e.getLocalizedMessage());
          }
          return morDc;
      }
  
      private VirtualMachineMO getVirtualMachineMO(String vmName, VmwareHypervisorHost hyperHost) {
          VirtualMachineMO vmMo;
          try {
              // find VM through datacenter (VM is not at the target host yet)
              vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
          } catch (Exception e) {
              String msg = "exception while searching for VM " + vmName + " in VMware datacenter";
              logger.error(msg);
              throw new CloudRuntimeException(msg + ": " + e.getLocalizedMessage());
          }
          return vmMo;
      }
  
      protected Answer execute(MigrateCommand cmd) {
          final String vmName = cmd.getVmName();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              if (hyperHost == null) {
                  throw new CloudRuntimeException("no hypervisor host found for migrate command");
              }
              ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter();
              if (morDc == null) {
                  throw new CloudRuntimeException("no Managed Object Reference for the Data Center found for migrate command");
              }
  
              // find VM through datacenter (VM is not at the target host yet)
              VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              if (vmMo == null) {
                  String msg = "VM " + vmName + " does not exist in VMware datacenter";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              VmwareHypervisorHost destHyperHost = getTargetHyperHost(new DatacenterMO(hyperHost.getContext(), morDc), cmd.getDestinationIp());
              if (destHyperHost == null) {
                  throw new CloudRuntimeException("no destination Hypervisor Host found for migrate command");
              }
  
              ManagedObjectReference morTargetPhysicalHost = destHyperHost.findMigrationTarget(vmMo);
              if (morTargetPhysicalHost == null) {
                  throw new Exception("Unable to find a target capable physical host");
              }
  
              if (!vmMo.migrate(destHyperHost.getHyperHostOwnerResourcePool(), morTargetPhysicalHost)) {
                  throw new Exception("Migration failed");
              }
  
              return new MigrateAnswer(cmd, true, "migration succeeded", null);
          } catch (Exception e) {
              logger.info(String.format("migrate command for %s failed due to %s", vmName, e.getLocalizedMessage()));
              return new MigrateAnswer(cmd, false, createLogMessageException(e, cmd), null);
          }
      }
  
      protected Answer execute(MigrateWithStorageCommand cmd) {
          final VirtualMachineTO vmTo = cmd.getVirtualMachine();
          final List<Pair<VolumeTO, StorageFilerTO>> volToFiler = cmd.getVolumeToFilerAsList();
          final String targetHost = cmd.getTargetHost();
  
          try {
              List<VolumeObjectTO> volumeToList =  relocateVirtualMachine(null, null, vmTo, targetHost, null, null, volToFiler);
              return new MigrateWithStorageAnswer(cmd, volumeToList);
          } catch (Throwable e) {
              String msg = "MigrateWithStorageCommand failed due to " + VmwareHelper.getExceptionMessage(e);
              logger.warn(msg, e);
              return new MigrateWithStorageAnswer(cmd, (Exception)e);
          }
      }
  
      private Answer migrateVolume(MigrateVolumeCommand cmd) {
          Answer answer;
          String path = cmd.getVolumePath();
  
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          VirtualMachineMO vmMo = null;
          DatastoreMO sourceDsMo;
          DatastoreMO destinationDsMo = null;
          ManagedObjectReference morSourceDS;
          ManagedObjectReference morDestinationDS;
          String vmdkDataStorePath;
          boolean isvVolsInvolved = false;
  
          String vmName = null;
          try {
              // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here
              // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway
              // we need to spawn a worker VM to attach the volume to and move it
              morSourceDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getSourcePool().getUuid());
              sourceDsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS);
              VmwareHypervisorHost hyperHostInTargetCluster = VmwareHelper.getHostMOFromHostName(getServiceContext(),
                      cmd.getHostGuidInTargetCluster());
              VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster;
              String targetDsName = cmd.getTargetPool().getUuid();
              morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName);
              if (morDestinationDS == null) {
                  String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName();
                  logger.error(msg);
                  throw new CloudRuntimeException(msg);
              }
              destinationDsMo = new DatastoreMO(hyperHost.getContext(), morDestinationDS);
  
              vmName = getWorkerName(getServiceContext(), cmd, 0, sourceDsMo);
              if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
                  isvVolsInvolved = true;
                  vmName = getWorkerName(getServiceContext(), cmd, 0, destinationDsMo);
              }
  
              // OfflineVmwareMigration: refactor for re-use
              // OfflineVmwareMigration: 1. find data(store)
              // OfflineVmwareMigration: more robust would be to find the store given the volume as it might have been moved out of band or due to error
              // example: DatastoreMO existingVmDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName()));
  
              logger.info("Create worker VM " + vmName);
              // OfflineVmwareMigration: 2. create the worker with access to the data(store)
              vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, sourceDsMo, vmName,
                      HypervisorHostHelper.getMinimumHostHardwareVersion(hyperHost, hyperHostInTargetCluster));
              if (vmMo == null) {
                  // OfflineVmwareMigration: don't throw a general Exception but think of a specific one
                  throw new CloudRuntimeException("Unable to create a worker VM for volume operation");
              }
  
              synchronized (this) {
                  // OfflineVmwareMigration: 3. attach the disk to the worker
                  String vmdkFileName = path + VMDK_EXTENSION;
                  vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(sourceDsMo, vmdkFileName);
                  if (!sourceDsMo.fileExists(vmdkDataStorePath)) {
                      if (logger.isDebugEnabled()) {
                          logger.debug(String.format("path not found (%s), trying under '%s'", vmdkFileName, path));
                      }
                      vmdkDataStorePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(sourceDsMo, path, vmdkFileName);
                  }
                  if (!sourceDsMo.folderExists(String.format("[%s]", sourceDsMo.getName()), path) || !sourceDsMo.fileExists(vmdkDataStorePath)) {
                      if (logger.isDebugEnabled()) {
                          logger.debug(String.format("path not found (%s), trying under '%s'", vmdkFileName, vmName));
                      }
                      vmdkDataStorePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(sourceDsMo, vmName, vmdkFileName);
                  }
                  if (!sourceDsMo.folderExists(String.format("[%s]", sourceDsMo.getName()), vmName) || !sourceDsMo.fileExists(vmdkDataStorePath)) {
                      vmdkDataStorePath = sourceDsMo.searchFileInSubFolders(vmdkFileName, true, null);
                  }
  
                  if (logger.isDebugEnabled()) {
                      logger.debug(String.format("attaching %s to %s for migration", vmdkDataStorePath, vmMo.getVmName()));
                  }
                  vmMo.attachDisk(new String[]{vmdkDataStorePath}, morSourceDS);
              }
  
              // OfflineVmwareMigration: 4. find the (worker-) VM
              // find VM through datacenter (VM is not at the target host yet)
              vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              if (vmMo == null) {
                  String msg = "VM " + vmName + " does not exist in VMware datacenter";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              if (logger.isTraceEnabled()) {
                  VirtualDisk[] disks = vmMo.getAllDiskDevice();
                  String format = "disk %d is attached as %s";
                  for (VirtualDisk disk : disks) {
                      logger.trace(String.format(format, disk.getKey(), vmMo.getVmdkFileBaseName(disk)));
                  }
              }
  
              // OfflineVmwareMigration: 5. create a relocate spec and perform
              Pair<VirtualDisk, String> vdisk = vmMo.getDiskDevice(path);
              if (vdisk == null) {
                  if (logger.isTraceEnabled())
                      logger.trace("migrate volume done (failed)");
                  throw new CloudRuntimeException("No such disk device: " + path);
              }
  
              VirtualDisk disk = vdisk.first();
              String vmdkAbsFile = VmwareHelper.getAbsoluteVmdkFile(disk);
              if (vmdkAbsFile != null && !vmdkAbsFile.isEmpty()) {
                  vmMo.updateAdapterTypeIfRequired(vmdkAbsFile);
              }
  
              // OfflineVmwareMigration: this may have to be disected and executed in separate steps
              answer = migrateAndAnswer(vmMo, cmd.getTargetPool().getUuid(), hyperHost, cmd);
          } catch (Exception e) {
              String msg = String.format("Migration of volume '%s' failed due to %s", cmd.getVolumePath(), e.getLocalizedMessage());
              logger.error(msg, e);
              answer = new Answer(cmd, false, msg);
          } finally {
              try {
                  // OfflineVmwareMigration: worker *may* have been renamed
                  vmName = vmMo.getVmName();
                  logger.info("Dettaching disks before destroying worker VM '" + vmName + "' after volume migration");
                  VirtualDisk[] disks = vmMo.getAllDiskDevice();
                  String format = "disk %d was migrated to %s";
                  for (VirtualDisk disk : disks) {
                      if (logger.isTraceEnabled()) {
                          logger.trace(String.format(format, disk.getKey(), vmMo.getVmdkFileBaseName(disk)));
                      }
                      vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(destinationDsMo, vmMo.getVmdkFileBaseName(disk) + VMDK_EXTENSION);
                      vmMo.detachDisk(vmdkDataStorePath, false);
                  }
                  logger.info("Destroy worker VM '" + vmName + "' after volume migration");
                  vmMo.destroy();
              } catch (Throwable e) {
                  logger.info("Failed to destroy worker VM: " + vmName);
              }
          }
          if (answer instanceof MigrateVolumeAnswer) {
              if (!isvVolsInvolved) {
                  String newPath = ((MigrateVolumeAnswer) answer).getVolumePath();
                  String vmdkFileName = newPath + VMDK_EXTENSION;
                  try {
                      VmwareStorageLayoutHelper.syncVolumeToRootFolder(destinationDsMo.getOwnerDatacenter().first(), destinationDsMo, newPath, vmName);
                      vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(destinationDsMo, vmdkFileName);
  
                      if (!destinationDsMo.fileExists(vmdkDataStorePath)) {
                          String msg = String.format("Migration of volume '%s' failed; file (%s) not found as path '%s'", cmd.getVolumePath(), vmdkFileName, vmdkDataStorePath);
                          logger.error(msg);
                          answer = new Answer(cmd, false, msg);
                      }
                  } catch (Exception e) {
                      String msg = String.format("Migration of volume '%s' failed due to %s", cmd.getVolumePath(), e.getLocalizedMessage());
                      logger.error(msg, e);
                      answer = new Answer(cmd, false, msg);
                  }
              }
          }
          return answer;
      }
  
      // OfflineVmwareMigration: refactor to be able to handle a detached volume
      private Answer execute(MigrateVolumeCommand cmd) {
          String volumePath = cmd.getVolumePath();
          String chainInfo = cmd.getChainInfo();
          StorageFilerTO poolTo = cmd.getPool();
          VolumeObjectTO volumeObjectTO = (VolumeObjectTO)cmd.getSrcData();
  
          String vmName = cmd.getAttachedVmName();
  
          VirtualMachineMO vmMo;
          VmwareHypervisorHost srcHyperHost;
  
          // OfflineVmwareMigration: ifhost is null ???
          if (StringUtils.isBlank(cmd.getAttachedVmName())) {
              return migrateVolume(cmd);
          }
          ManagedObjectReference morDs;
          ManagedObjectReference morDc;
          VirtualMachineRelocateSpec relocateSpec = new VirtualMachineRelocateSpec();
          List<VirtualMachineRelocateSpecDiskLocator> diskLocators = new ArrayList<>();
          VirtualMachineRelocateSpecDiskLocator diskLocator;
  
          String tgtDsName;
  
          try {
              srcHyperHost = getHyperHost(getServiceContext());
              morDc = srcHyperHost.getHyperHostDatacenter();
              tgtDsName = poolTo.getUuid();
  
              // find VM in this datacenter not just in this cluster.
              DatacenterMO dcMo = new DatacenterMO(getServiceContext(), morDc);
              vmMo = dcMo.findVm(vmName);
  
              if (vmMo == null) {
                  String msg = "VM " + vmName + " does not exist in VMware datacenter " + morDc.getValue();
                  logger.error(msg);
                  throw new CloudRuntimeException(msg);
              }
              vmName = vmMo.getName();
              morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, tgtDsName);
              if (morDs == null) {
                  String msg = "Unable to find the mounted datastore with name: " + tgtDsName + " on source host: " + srcHyperHost.getHyperHostName()
                          + " to execute MigrateVolumeCommand";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              DatastoreMO targetDsMo = new DatastoreMO(srcHyperHost.getContext(), morDs);
              if (cmd.getContextParam(DiskTO.PROTOCOL_TYPE) != null && cmd.getContextParam(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) {
                  VmwareContext context = getServiceContext();
                  VmwareHypervisorHost hyperHost = getHyperHost(context);
                  VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
                  VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volumePath, chainInfo, false, null, poolTo.getUuid(), hyperHost, context);
                  if (diskInfoBuilder != null && matchingExistingDisk != null) {
                      String[] diskChain = matchingExistingDisk.getDiskChain();
                      DatastoreFile file = new DatastoreFile(diskChain[0]);
                      if (!file.getFileBaseName().equalsIgnoreCase(volumePath)) {
                          if (logger.isInfoEnabled())
                              logger.info("Detected disk-chain top file change on volume: " + volumePath + " -> " + file.getFileBaseName());
                          volumePath = file.getFileBaseName();
                      }
                  }
              }
  
              String fullVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(targetDsMo, vmName, volumePath + VMDK_EXTENSION);
              Pair<VirtualDisk, String> diskInfo = getVirtualDiskInfo(vmMo, appendFileType(volumePath, VMDK_EXTENSION));
              String vmdkAbsFile = VmwareHelper.getAbsoluteVmdkFile(diskInfo.first());
              if (vmdkAbsFile != null && !vmdkAbsFile.isEmpty()) {
                  vmMo.updateAdapterTypeIfRequired(vmdkAbsFile);
              }
              int diskId = diskInfo.first().getKey();
  
              diskLocator = new VirtualMachineRelocateSpecDiskLocator();
              diskLocator.setDatastore(morDs);
              diskLocator.setDiskId(diskId);
              diskLocators.add(diskLocator);
              if (cmd.getVolumeType() == Volume.Type.ROOT) {
                  relocateSpec.setDatastore(morDs);
                  // If a target datastore is provided for the VM, then by default all volumes associated with the VM will be migrated to that target datastore.
                  // Hence set the existing datastore as target datastore for volumes that are not to be migrated.
                  List<Pair<Integer, ManagedObjectReference>> diskDatastores = vmMo.getAllDiskDatastores();
                  for (Pair<Integer, ManagedObjectReference> diskDatastore : diskDatastores) {
                      if (diskDatastore.first() != diskId) {
                          diskLocator = new VirtualMachineRelocateSpecDiskLocator();
                          diskLocator.setDiskId(diskDatastore.first());
                          diskLocator.setDatastore(diskDatastore.second());
                          diskLocators.add(diskLocator);
                      }
                  }
              }
  
              relocateSpec.getDisk().addAll(diskLocators);
  
              // Change datastore
              if (!vmMo.changeDatastore(relocateSpec)) {
                  throw new Exception("Change datastore operation failed during volume migration");
              } else {
                  logger.debug("Successfully migrated volume " + volumePath + " to target datastore " + tgtDsName);
              }
  
              // Consolidate VM disks.
              // In case of a linked clone VM, if VM's disks are not consolidated,
              // further volume operations on the ROOT volume such as volume snapshot etc. will result in DB inconsistencies.
              if (!vmMo.consolidateVmDisks()) {
                  logger.warn("VM disk consolidation failed after storage migration.");
              } else {
                  logger.debug("Successfully consolidated disks of VM " + vmName + ".");
              }
  
              // Update and return volume path and chain info because that could have changed after migration
              if (!targetDsMo.fileExists(fullVolumePath)) {
                  VirtualDisk[] disks = vmMo.getAllDiskDevice();
                  for (VirtualDisk disk : disks)
                      if (disk.getKey() == diskId) {
                          volumePath = vmMo.getVmdkFileBaseName(disk);
                      }
              }
              VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
              chainInfo = _gson.toJson(diskInfoBuilder.getDiskInfoByBackingFileBaseName(volumePath, targetDsMo.getName()));
              MigrateVolumeAnswer answer = new MigrateVolumeAnswer(cmd, true, null, volumePath);
              answer.setVolumeChainInfo(chainInfo);
              return answer;
          } catch (Exception e) {
              String msg = "Catch Exception " + e.getClass().getName() + " due to " + e.getMessage();
              logger.error(msg, e);
              return new MigrateVolumeAnswer(cmd, false, e.getMessage(), null);
          }
      }
  
      private Pair<VirtualDisk, String> getVirtualDiskInfo(VirtualMachineMO vmMo, String srcDiskName) throws Exception {
          Pair<VirtualDisk, String> deviceInfo = vmMo.getDiskDevice(srcDiskName);
          if (deviceInfo == null) {
              throw new Exception("No such disk device: " + srcDiskName);
          }
          return deviceInfo;
      }
  
      private VmwareHypervisorHost getTargetHyperHost(DatacenterMO dcMo, String destIp) throws Exception {
  
          VmwareManager mgr = dcMo.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
          List<ObjectContent> ocs = dcMo.getHostPropertiesOnDatacenterHostFolder(new String[]{"name", "parent"});
          if (ocs != null && ocs.size() > 0) {
              for (ObjectContent oc : ocs) {
                  HostMO hostMo = new HostMO(dcMo.getContext(), oc.getObj());
                  VmwareHypervisorHostNetworkSummary netSummary = hostMo.getHyperHostNetworkSummary(mgr.getManagementPortGroupByHost(hostMo));
                  if (destIp.equalsIgnoreCase(netSummary.getHostIp())) {
                      return new HostMO(dcMo.getContext(), oc.getObj());
                  }
              }
          }
  
          throw new Exception("Unable to locate dest host by " + destIp);
      }
  
      protected Answer execute(CreateStoragePoolCommand cmd) {
          if (cmd.getCreateDatastore()) {
              try {
                  VmwareContext context = getServiceContext();
  
                  _storageProcessor.prepareManagedDatastore(context, getHyperHost(context), cmd.getDetails().get(CreateStoragePoolCommand.DATASTORE_NAME),
                          cmd.getDetails().get(CreateStoragePoolCommand.IQN), cmd.getDetails().get(CreateStoragePoolCommand.STORAGE_HOST),
                          Integer.parseInt(cmd.getDetails().get(CreateStoragePoolCommand.STORAGE_PORT)));
              } catch (Exception ex) {
                  return new Answer(cmd, false, "Issue creating datastore");
              }
          }
  
          return new Answer(cmd, true, "success");
      }
  
      protected Answer execute(ModifyTargetsCommand cmd) {
          VmwareContext context = getServiceContext(cmd);
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          List<HostMO> hostMOs = new ArrayList<>();
  
          if (cmd.getApplyToAllHostsInCluster()) {
              try {
                  ManagedObjectReference morCluster = hyperHost.getHyperHostCluster();
                  ClusterMO clusterMO = new ClusterMO(context, morCluster);
  
                  List<Pair<ManagedObjectReference, String>> hosts = clusterMO.getClusterHosts();
  
                  for (Pair<ManagedObjectReference, String> host : hosts) {
                      HostMO hostMO = new HostMO(context, host.first());
  
                      hostMOs.add(hostMO);
                  }
              } catch (Exception ex) {
                  logger.error(ex.getMessage(), ex);
  
                  throw new CloudRuntimeException(ex.getMessage(), ex);
              }
          } else {
              hostMOs.add((HostMO) hyperHost);
          }
  
          handleTargets(cmd.getAdd(), cmd.getTargetTypeToRemove(), cmd.isRemoveAsync(), cmd.getTargets(), hostMOs);
  
          return new ModifyTargetsAnswer();
      }
  
      protected Answer execute(ModifyStoragePoolCommand cmd) {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              StorageFilerTO pool = cmd.getPool();
  
              if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS && pool.getType() != StoragePoolType.PreSetup && pool.getType() != StoragePoolType.DatastoreCluster) {
                  throw new Exception("Unsupported storage pool type " + pool.getType());
              }
  
              ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid());
  
              if (morDatastore == null) {
                  morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup || pool.getType() == StoragePoolType.DatastoreCluster), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true);
              }
  
              assert (morDatastore != null);
  
              DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore);
              HypervisorHostHelper.createBaseFolder(dsMo, hyperHost, pool.getType());
  
              long capacity;
              long available;
              List<ModifyStoragePoolAnswer> childDatastoresModifyStoragePoolAnswers = new ArrayList<>();
              if (pool.getType() == StoragePoolType.DatastoreCluster) {
                  StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDatastore);
                  StoragePodSummary dsClusterSummary = datastoreClusterMo.getDatastoreClusterSummary();
                  capacity = dsClusterSummary.getCapacity();
                  available = dsClusterSummary.getFreeSpace();
  
                  List<ManagedObjectReference> childDatastoreMors = datastoreClusterMo.getDatastoresInDatastoreCluster();
                  for (ManagedObjectReference childDsMor : childDatastoreMors) {
                      DatastoreMO childDsMo = new DatastoreMO(getServiceContext(), childDsMor);
  
                      Map<String, TemplateProp> tInfo = new HashMap<>();
                      DatastoreSummary summary = childDsMo.getDatastoreSummary();;
                      ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, summary.getCapacity(), summary.getFreeSpace(), tInfo);
                      StoragePoolInfo poolInfo = answer.getPoolInfo();
                      poolInfo.setName(summary.getName());
                      String datastoreClusterPath = pool.getPath();
                      int pathstartPosition = datastoreClusterPath.lastIndexOf('/');
                      String datacenterName = datastoreClusterPath.substring(0, pathstartPosition+1);
                      String childPath = datacenterName + summary.getName();
                      poolInfo.setHostPath(childPath);
                      String uuid = childDsMo.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID);
                      if (uuid == null || !uuid.contains("-")) {
                          uuid = UuidUtils.nameUUIDFromBytes(((pool.getHost() + childPath)).getBytes()).toString();
                      }
                      poolInfo.setUuid(uuid);
                      poolInfo.setLocalPath(cmd.LOCAL_PATH_PREFIX + File.separator + uuid);
                      answer.setPoolInfo(poolInfo);
                      answer.setPoolType(summary.getType());
                      answer.setLocalDatastoreName(morDatastore.getValue());
  
                      childDsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, uuid);
                      HypervisorHostHelper.createBaseFolderInDatastore(childDsMo, hyperHost.getHyperHostDatacenter());
  
                      childDatastoresModifyStoragePoolAnswers.add(answer);
                  }
              } else {
                  HypervisorHostHelper.createBaseFolderInDatastore(dsMo, hyperHost.getHyperHostDatacenter());
  
                  DatastoreSummary summary = dsMo.getDatastoreSummary();
                  capacity = summary.getCapacity();
                  available = summary.getFreeSpace();
              }
  
              Map<String, TemplateProp> tInfo = new HashMap<>();
              ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo);
              answer.setDatastoreClusterChildren(childDatastoresModifyStoragePoolAnswers);
  
              if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup) && pool.getType() != StoragePoolType.DatastoreCluster) {
                  answer.setPoolType(dsMo.getDatastoreType());
                  answer.setLocalDatastoreName(morDatastore.getValue());
              }
  
              return answer;
          } catch (Throwable e) {
              return new Answer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(GetStoragePoolCapabilitiesCommand cmd) {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
              HostMO host = (HostMO) hyperHost;
  
              StorageFilerTO pool = cmd.getPool();
  
              ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid());
  
              if (morDatastore == null) {
                  morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup || pool.getType() == StoragePoolType.DatastoreCluster), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true);
              }
  
              assert (morDatastore != null);
  
              DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore);
  
              GetStoragePoolCapabilitiesAnswer answer = new GetStoragePoolCapabilitiesAnswer(cmd);
  
              boolean hardwareAccelerationSupportForDataStore = getHardwareAccelerationSupportForDataStore(host.getMor(), dsMo.getName());
              Map<String, String> poolDetails = answer.getPoolDetails();
              poolDetails.put(Storage.Capability.HARDWARE_ACCELERATION.toString(), String.valueOf(hardwareAccelerationSupportForDataStore));
              answer.setPoolDetails(poolDetails);
              answer.setResult(true);
  
              return answer;
          } catch (Throwable e) {
              GetStoragePoolCapabilitiesAnswer answer = new GetStoragePoolCapabilitiesAnswer(cmd);
              answer.setResult(false);
              answer.setDetails(createLogMessageException(e, cmd));
              return answer;
          }
      }
  
      private boolean getHardwareAccelerationSupportForDataStore(ManagedObjectReference host, String dataStoreName) throws Exception {
          HostConfigInfo config = getServiceContext().getVimClient().getDynamicProperty(host, "config");
          List<HostFileSystemMountInfo> mountInfoList = config.getFileSystemVolume().getMountInfo();
          for (HostFileSystemMountInfo hostFileSystemMountInfo: mountInfoList) {
              if ( hostFileSystemMountInfo.getVolume().getName().equals(dataStoreName) ) {
                  return hostFileSystemMountInfo.getVStorageSupport().equals("vStorageSupported");
              }
          }
          return false;
      }
  
      private void handleTargets(boolean add, ModifyTargetsCommand.TargetTypeToRemove targetTypeToRemove, boolean isRemoveAsync,
                                 List<Map<String, String>> targets, List<HostMO> hosts) {
          if (targets != null && targets.size() > 0) {
              try {
                  _storageProcessor.handleTargets(add, targetTypeToRemove, isRemoveAsync, targets, hosts);
              } catch (Exception ex) {
                  logger.warn(ex.getMessage());
              }
          }
      }
  
      protected Answer execute(DeleteStoragePoolCommand cmd) {
          try {
              if (cmd.getRemoveDatastore()) {
                  _storageProcessor.handleDatastoreAndVmdkDetach(cmd, cmd.getDetails().get(DeleteStoragePoolCommand.DATASTORE_NAME),
                          cmd.getDetails().get(DeleteStoragePoolCommand.IQN), cmd.getDetails().get(DeleteStoragePoolCommand.STORAGE_HOST),
                          Integer.parseInt(cmd.getDetails().get(DeleteStoragePoolCommand.STORAGE_PORT)));
  
                  return new Answer(cmd, true, "success");
              } else {
                  // We will leave datastore cleanup management to vCenter. Since for cluster VMFS datastore, it will always
                  // be mounted by vCenter.
  
                  // VmwareHypervisorHost hyperHost = this.getHyperHost(getServiceContext());
                  // hyperHost.unmountDatastore(pool.getUuid());
  
                  return new Answer(cmd, true, "success");
              }
          } catch (Throwable e) {
              if (e instanceof RemoteException) {
                  logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
  
                  invalidateServiceContext();
              }
  
              StorageFilerTO pool = cmd.getPool();
              String msg = String.format("DeleteStoragePoolCommand (pool: [%s], path: [%s]) failed due to [%s].", pool.getHost(), pool.getPath(), VmwareHelper.getExceptionMessage(e));
              logger.error(msg, e);
  
              return new Answer(cmd, false, msg);
          }
      }
  
      public static String getDatastoreName(String str) {
          return str.replace('/', '-');
      }
  
      public static String createDatastoreNameFromIqn(String iqn) {
          return "-" + iqn + "-0";
      }
  
      protected AttachIsoAnswer execute(AttachIsoCommand cmd) {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
              VirtualMachineMO vmMo = HypervisorHostHelper.findVmOnHypervisorHostOrPeer(hyperHost, cmd.getVmName());
              if (vmMo == null) {
                  String msg = "Unable to find VM in vSphere to execute AttachIsoCommand, vmName: " + cmd.getVmName();
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              String storeUrl = cmd.getStoreUrl();
              if (storeUrl == null) {
                  if (!cmd.getIsoPath().equalsIgnoreCase(TemplateManager.VMWARE_TOOLS_ISO)) {
                      String msg = "ISO store root url is not found in AttachIsoCommand";
                      logger.error(msg);
                      throw new Exception(msg);
                  } else {
                      if (cmd.isAttach()) {
                          vmMo.mountToolsInstaller();
                      } else {
                          try {
                              if (!vmMo.unmountToolsInstaller()) {
                                  return new AttachIsoAnswer(cmd, false,
                                          "Failed to unmount vmware-tools installer ISO as the corresponding CDROM device is locked by VM. Please unmount the CDROM device inside the VM and ret-try.");
                              }
                          } catch (Throwable e) {
                              vmMo.detachIso(null, cmd.isForce());
                          }
                      }
  
                      return new AttachIsoAnswer(cmd);
                  }
              }
  
              ManagedObjectReference morSecondaryDs = prepareSecondaryDatastoreOnHost(storeUrl);
              String isoPath = cmd.getIsoPath();
              if (!isoPath.startsWith(storeUrl)) {
                  assert (false);
                  String msg = "ISO path does not start with the secondary storage root";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              int isoNameStartPos = isoPath.lastIndexOf('/');
              String isoFileName = isoPath.substring(isoNameStartPos + 1);
              String isoStorePathFromRoot = isoPath.substring(storeUrl.length() + 1, isoNameStartPos + 1);
  
  
              // TODO, check if iso is already attached, or if there is a previous
              // attachment
              DatastoreMO secondaryDsMo = new DatastoreMO(getServiceContext(), morSecondaryDs);
              String storeName = secondaryDsMo.getName();
              String isoDatastorePath = String.format("[%s] %s%s", storeName, isoStorePathFromRoot, isoFileName);
  
              if (cmd.isAttach()) {
                  vmMo.attachIso(isoDatastorePath, morSecondaryDs, true, false, cmd.getDeviceKey(), cmd.isForce());
                  return new AttachIsoAnswer(cmd);
              } else {
                  int key = vmMo.detachIso(isoDatastorePath, cmd.isForce());
                  return new AttachIsoAnswer(cmd, key);
              }
  
          } catch (Throwable e) {
              if (e instanceof RemoteException) {
                  logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
                  invalidateServiceContext();
              }
  
              String message = String.format("AttachIsoCommand(%s) failed due to [%s].", cmd.isAttach()? "attach" : "detach", VmwareHelper.getExceptionMessage(e));
              logger.error(message, e);
              return new AttachIsoAnswer(cmd, false, message);
          }
      }
  
      public synchronized ManagedObjectReference prepareSecondaryDatastoreOnHost(String storeUrl) throws Exception {
          String storeName = getSecondaryDatastoreUUID(storeUrl);
          URI uri = new URI(storeUrl);
  
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false);
  
          if (morDatastore == null)
              throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl);
  
          return morDatastore;
      }
  
      public synchronized ManagedObjectReference prepareSecondaryDatastoreOnSpecificHost(String storeUrl, VmwareHypervisorHost hyperHost) throws Exception {
          String storeName = getSecondaryDatastoreUUID(storeUrl);
          URI uri = new URI(storeUrl);
  
          ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false);
  
          if (morDatastore == null)
              throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl);
  
          return morDatastore;
      }
  
      private String getSecondaryDatastoreUUID(String storeUrl) {
          String uuid = null;
          try {
              uuid = UuidUtils.nameUUIDFromBytes(storeUrl.getBytes("UTF-8")).toString();
          } catch (UnsupportedEncodingException e) {
              logger.warn("Failed to create UUID from string " + storeUrl + ". Bad storeUrl or UTF-8 encoding error.");
          }
          return uuid;
      }
  
      protected Answer execute(ValidateSnapshotCommand cmd) {
          // the command is no longer available
          String expectedSnapshotBackupUuid = null;
          String actualSnapshotBackupUuid = null;
          String actualSnapshotUuid = null;
          return new ValidateSnapshotAnswer(cmd, false, "ValidateSnapshotCommand is not supported for vmware yet", expectedSnapshotBackupUuid, actualSnapshotBackupUuid,
                  actualSnapshotUuid);
      }
  
      protected Answer execute(ManageSnapshotCommand cmd) {
          long snapshotId = cmd.getSnapshotId();
  
          /*
           * "ManageSnapshotCommand",
           * "{\"_commandSwitch\":\"-c\",\"_volumePath\":\"i-2-3-KY-ROOT\",\"_snapshotName\":\"i-2-3-KY_i-2-3-KY-ROOT_20101102203827\",\"_snapshotId\":1,\"_vmName\":\"i-2-3-KY\"}"
           */
          boolean success;
          String cmdSwitch = cmd.getCommandSwitch();
          String snapshotOp = "Unsupported snapshot command." + cmdSwitch;
          if (cmdSwitch.equals(ManageSnapshotCommand.CREATE_SNAPSHOT)) {
              snapshotOp = "create";
          } else if (cmdSwitch.equals(ManageSnapshotCommand.DESTROY_SNAPSHOT)) {
              snapshotOp = "destroy";
          }
  
          String details;
          String snapshotUUID;
  
          // snapshot operation (create or destroy) is handled inside BackupSnapshotCommand(), we just fake
          // a success return here
          snapshotUUID = UUID.randomUUID().toString();
          success = true;
          details = null;
  
          return new ManageSnapshotAnswer(cmd, snapshotId, snapshotUUID, success, details);
      }
  
      protected Answer execute(BackupSnapshotCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              return mgr.getStorageManager().execute(this, cmd);
          } catch (Throwable e) {
              return new BackupSnapshotAnswer(cmd, false, createLogMessageException(e, cmd), null, true);
          }
      }
  
      protected Answer execute(CreateVMSnapshotCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              return mgr.getStorageManager().execute(this, cmd);
          } catch (Exception e) {
              createLogMessageException(e, cmd);
              return new CreateVMSnapshotAnswer(cmd, false, "");
          }
      }
  
      protected Answer execute(DeleteVMSnapshotCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              return mgr.getStorageManager().execute(this, cmd);
          } catch (Exception e) {
              createLogMessageException(e, cmd);
              return new DeleteVMSnapshotAnswer(cmd, false, "");
          }
      }
  
      protected Answer execute(RevertToVMSnapshotCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              return mgr.getStorageManager().execute(this, cmd);
          } catch (Exception e) {
              createLogMessageException(e, cmd);
              return new RevertToVMSnapshotAnswer(cmd, false, "");
          }
      }
  
      protected Answer execute(CreateVolumeFromSnapshotCommand cmd) {
          String details;
          boolean success = false;
          String newVolumeName = UUID.randomUUID().toString();
  
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              return mgr.getStorageManager().execute(this, cmd);
          } catch (Throwable e) {
              details = createLogMessageException(e, cmd);
          }
  
          return new CreateVolumeFromSnapshotAnswer(cmd, success, details, newVolumeName);
      }
  
      protected Answer execute(CreatePrivateTemplateFromVolumeCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              return mgr.getStorageManager().execute(this, cmd);
  
          } catch (Throwable e) {
              return new CreatePrivateTemplateAnswer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(final UpgradeSnapshotCommand cmd) {
          return new Answer(cmd, true, "success");
      }
  
      protected Answer execute(CreatePrivateTemplateFromSnapshotCommand cmd) {
          try {
              VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              return mgr.getStorageManager().execute(this, cmd);
  
          } catch (Throwable e) {
              return new CreatePrivateTemplateAnswer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(GetStorageStatsCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getStorageId());
  
              if (morDs != null) {
                  long capacity;
                  long free;
                  if (cmd.getPooltype() == StoragePoolType.DatastoreCluster) {
                      StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDs);
                      StoragePodSummary summary = datastoreClusterMo.getDatastoreClusterSummary();
                      capacity = summary.getCapacity();
                      free = summary.getFreeSpace();
                  } else {
                      DatastoreMO datastoreMo = new DatastoreMO(context, morDs);
                      DatastoreSummary summary = datastoreMo.getDatastoreSummary();
                      capacity = summary.getCapacity();
                      free = summary.getFreeSpace();
                  }
  
                  long used = capacity - free;
  
                  logger.debug(String.format("Datastore summary info: [storageId: %s, ], localPath: %s, poolType: %s, capacity: %s, free: %s, used: %s].", cmd.getStorageId(),
                          cmd.getLocalPath(), cmd.getPooltype(), toHumanReadableSize(capacity), toHumanReadableSize(free), toHumanReadableSize(used)));
  
                  if (capacity <= 0) {
                      logger.warn("Something is wrong with vSphere NFS datastore, rebooting ESX(ESXi) host should help");
                  }
  
                  return new GetStorageStatsAnswer(cmd, capacity, used);
              } else {
                  String msg = String.format("Could not find datastore for GetStorageStatsCommand: [storageId: %s, localPath: %s, poolType: %s].",
                          cmd.getStorageId(), cmd.getLocalPath(), cmd.getPooltype());
  
                  logger.error(msg);
                  return new GetStorageStatsAnswer(cmd, msg);
              }
          } catch (Throwable e) {
              if (e instanceof RemoteException) {
                  logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
                  invalidateServiceContext();
              }
  
              String msg = String.format("Unable to execute GetStorageStatsCommand(storageId : [%s], localPath: [%s], poolType: [%s]) due to [%s]", cmd.getStorageId(), cmd.getLocalPath(), cmd.getPooltype(), VmwareHelper.getExceptionMessage(e));
              logger.error(msg, e);
              return new GetStorageStatsAnswer(cmd, msg);
          }
      }
  
      protected Answer execute(GetVncPortCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              assert (hyperHost instanceof HostMO);
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getName());
              if (vmMo == null) {
                  if (logger.isDebugEnabled()) {
                      logger.debug("Unable to find the owner VM for GetVncPortCommand on host " + hyperHost.getHyperHostName() + ", try within datacenter");
                  }
  
                  vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getName());
  
                  if (vmMo == null) {
                      throw new Exception("Unable to find VM in vSphere, vm: " + cmd.getName());
                  }
              }
  
              Pair<String, Integer> portInfo = vmMo.getVncPort(mgr.getManagementPortGroupByHost((HostMO) hyperHost));
  
              if (logger.isTraceEnabled()) {
                  logger.trace("Found vnc port info. vm: " + cmd.getName() + " host: " + portInfo.first() + ", vnc port: " + portInfo.second());
              }
              return new GetVncPortAnswer(cmd, portInfo.first(), portInfo.second());
          } catch (Throwable e) {
              return new GetVncPortAnswer(cmd, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(SetupCommand cmd) {
          return new SetupAnswer(cmd, false);
      }
  
      protected Answer execute(MaintainCommand cmd) {
          return new MaintainAnswer(cmd, "Put host in maintenance");
      }
  
      protected Answer execute(PingTestCommand cmd) {
          String controlIp = cmd.getRouterIp();
          if (controlIp != null) {
              String args = " -c 1 -n -q " + cmd.getPrivateIp();
              try {
                  Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DefaultDomRSshPort, "root", getSystemVmKeyFile(), null, "/bin/ping" + args);
                  if (result.first())
                      return new Answer(cmd);
              } catch (Exception e) {
                  logger.error("Unable to execute ping command on DomR (" + controlIp + "), domR may not be ready yet. failure due to " + VmwareHelper.getExceptionMessage(e), e);
              }
              return new Answer(cmd, false, "PingTestCommand failed");
          } else {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              try {
                  HostMO hostMo = (HostMO) hyperHost;
                  ClusterMO clusterMo = new ClusterMO(context, hostMo.getHyperHostCluster());
                  VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
                  List<Pair<ManagedObjectReference, String>> hosts = clusterMo.getClusterHosts();
                  for (Pair<ManagedObjectReference, String> entry : hosts) {
                      HostMO hostInCluster = new HostMO(context, entry.first());
                      String hostIp = hostInCluster.getHostManagementIp(mgr.getManagementPortGroupName());
                      if (hostIp != null && hostIp.equals(cmd.getComputingHostIp())) {
                          if (hostInCluster.isHyperHostConnected())
                              return new Answer(cmd);
                          else
                              return new Answer(cmd, false, "PingTestCommand failed");
                      }
                  }
              } catch (Exception e) {
                  logger.error("Unable to execute ping command on host (" + cmd.getComputingHostIp() + "). failure due to " + VmwareHelper.getExceptionMessage(e), e);
              }
  
              return new Answer(cmd, false, "PingTestCommand failed");
          }
      }
  
      protected Answer execute(CheckOnHostCommand cmd) {
          return new CheckOnHostAnswer(cmd, null, "Not Implmeneted");
      }
  
      protected Answer execute(ModifySshKeysCommand cmd) {
          logger.debug(String.format("Executing resource command %s.", cmd.getClass().getSimpleName()));
          return new Answer(cmd);
      }
  
      protected Answer execute(GetVmIpAddressCommand cmd) {
          String details = "Unable to find IP Address of VM. ";
          String vmName = cmd.getVmName();
          boolean result = false;
          String ip;
          Answer answer;
  
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          if (vmName == null || vmName.isEmpty()) {
              details += "Name of instance provided is NULL or empty.";
              return new Answer(cmd, result, details);
          }
  
          try {
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
              if (vmMo != null) {
                  GuestInfo guestInfo = vmMo.getGuestInfo();
                  VirtualMachineToolsStatus toolsStatus = guestInfo.getToolsStatus();
                  if (toolsStatus == VirtualMachineToolsStatus.TOOLS_NOT_INSTALLED) {
                      details += "Vmware tools not installed.";
                  } else {
                      var normalizedMac = cmd.getMacAddress().replaceAll("-", ":");
                      for(var guestInfoNic : guestInfo.getNet()) {
                          var normalizedNicMac = guestInfoNic.getMacAddress().replaceAll("-", ":");
                          if (!result && normalizedNicMac.equalsIgnoreCase(normalizedMac)) {
                              result = true;
                              details = null;
                              for (var ipAddr : guestInfoNic.getIpAddress()) {
                                  if (NetUtils.isValidIp4(ipAddr) && (cmd.getVmNetworkCidr() == null || NetUtils.isIpWithInCidrRange(ipAddr, cmd.getVmNetworkCidr()))) {
                                      details = ipAddr;
                                  }
                              }
                              break;
                          }
                      }
                  }
              } else {
                  details += "VM " + vmName + " no longer exists on vSphere host: " + hyperHost.getHyperHostName();
                  logger.info(details);
              }
          } catch (Throwable e) {
              createLogMessageException(e, cmd);
              details = String.format("%s. Encountered exception: [%s].", details,  VmwareHelper.getExceptionMessage(e));
              logger.error(details);
          }
  
          answer = new Answer(cmd, result, details);
          if (logger.isTraceEnabled()) {
              logger.trace("Returning GetVmIpAddressAnswer: " + _gson.toJson(answer));
          }
          return answer;
      }
  
      @Override
      public PrimaryStorageDownloadAnswer execute(PrimaryStorageDownloadCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              return (PrimaryStorageDownloadAnswer) mgr.getStorageManager().execute(this, cmd);
          } catch (Throwable e) {
              return new PrimaryStorageDownloadAnswer(createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(PvlanSetupCommand cmd) {
          // Pvlan related operations are performed in the start/stop command paths
          // for vmware. This function is implemented to support mgmt layer code
          // that issue this command. Note that pvlan operations are supported only
          // in Distributed Virtual Switch environments for vmware deployments.
          return new Answer(cmd, true, "success");
      }
  
      protected Answer execute(CleanupVMCommand cmd) {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
  
          try {
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              if (vmMo == null) {
                  String msg = String.format("VM [%s] not found on vCenter, cleanup not needed.", cmd.getVmName());
                  logger.debug(msg);
                  return new Answer(cmd, true, msg);
              }
              vmMo.destroy();
              String msg = String.format("VM [%s] remnants on vCenter cleaned up.", cmd.getVmName());
              logger.debug(msg);
              return new Answer(cmd, true, msg);
          } catch (Exception e) {
              if (e.getMessage().contains("was not found")) {
                  String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName());
                  logger.debug(msg);
                  return new Answer(cmd, true, msg);
              }
              return new Answer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      protected Answer execute(UnregisterVMCommand cmd) {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          try {
              DatacenterMO dataCenterMo = new DatacenterMO(getServiceContext(), hyperHost.getHyperHostDatacenter());
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              if (vmMo != null) {
                  try {
                      VirtualMachineFileLayoutEx vmFileLayout = vmMo.getFileLayout();
                      context.getService().unregisterVM(vmMo.getMor());
                      if (cmd.getCleanupVmFiles()) {
                          deleteUnregisteredVmFiles(vmFileLayout, dataCenterMo, false, null);
                      }
                      return new Answer(cmd, true, "unregister succeeded");
                  } catch (Exception e) {
                      logger.warn("We are not able to unregister VM " + VmwareHelper.getExceptionMessage(e));
                  }
  
                  String msg = "Expunge failed in vSphere. vm: " + cmd.getVmName();
                  logger.warn(msg);
                  return new Answer(cmd, false, msg);
              } else {
                  String msg = "Unable to find the VM in vSphere to unregister, assume it is already removed. VM: " + cmd.getVmName();
                  logger.warn(msg);
                  return new Answer(cmd, true, msg);
              }
          } catch (Exception e) {
              return new Answer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      /**
       * UnregisterNicCommand is used to remove a portgroup created for this
       * specific nic. The portgroup will have the name set to the UUID of the
       * nic. Introduced to cleanup the portgroups created for each nic that is
       * plugged into an lswitch (Nicira NVP plugin)
       *
       * @param cmd
       * @return
       */
      protected Answer execute(UnregisterNicCommand cmd) {
          if (_guestTrafficInfo == null) {
              return new Answer(cmd, false, "No Guest Traffic Info found, unable to determine where to clean up");
          }
  
          try {
              if (_guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) {
                  // For now we only need to cleanup the nvp specific portgroups
                  // on the standard switches
                  return new Answer(cmd, true, "Nothing to do");
              }
  
              logger.debug("Cleaning up portgroup " + cmd.getNicUuid() + " on switch " + _guestTrafficInfo.getVirtualSwitchName());
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost host = getHyperHost(context);
              ManagedObjectReference clusterMO = host.getHyperHostCluster();
  
              // Get a list of all the hosts in this cluster
              @SuppressWarnings("unchecked")
              List<ManagedObjectReference> hosts = (List<ManagedObjectReference>) context.getVimClient().getDynamicProperty(clusterMO, "host");
              if (hosts == null) {
                  return new Answer(cmd, false, "No hosts in cluster, which is pretty weird");
              }
  
              for (ManagedObjectReference hostMOR : hosts) {
                  HostMO hostMo = new HostMO(context, hostMOR);
                  hostMo.deletePortGroup(cmd.getNicUuid().toString());
                  logger.debug("Removed portgroup " + cmd.getNicUuid() + " from host " + hostMo.getHostName());
              }
              return new Answer(cmd, true, "Unregistered resources for NIC " + cmd.getNicUuid());
          } catch (Exception e) {
              return new Answer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      public void cleanupNetwork(DatacenterMO dcMO, NetworkDetails netDetails) {
          if (!VmwareManager.s_vmwareCleanupPortGroups.value()){
              return;
          }
  
          try {
              synchronized(this) {
                  if (!areVMsOnNetwork(dcMO, netDetails)) {
                      cleanupPortGroup(dcMO, netDetails.getName());
                  }
              }
          } catch(Throwable e) {
              logger.warn("Unable to cleanup network due to exception: " + e.getMessage(), e);
          }
      }
  
      private void cleanupPortGroup(DatacenterMO dcMO, String portGroupName) throws Exception {
          if (StringUtils.isBlank(portGroupName)) {
              logger.debug("Unspecified network port group, couldn't cleanup");
              return;
          }
  
          List<HostMO> hosts = dcMO.getAllHostsOnDatacenter();
          if (!CollectionUtils.isEmpty(hosts)) {
              for (HostMO host : hosts) {
                  host.deletePortGroup(portGroupName);
              }
          }
      }
  
      private boolean areVMsOnNetwork(DatacenterMO dcMO, NetworkDetails netDetails) throws Exception {
          if (netDetails == null || netDetails.getName() == null) {
              throw new CloudRuntimeException("Unspecified network details / port group, couldn't check VMs on network port group");
          }
  
          List<HostMO> hosts = dcMO.getAllHostsOnDatacenter();
          if (!CollectionUtils.isEmpty(hosts)) {
              for (HostMO host : hosts) {
                  NetworkMO networkMo = new NetworkMO(host.getContext(), netDetails.getNetworkMor());
                  List<ManagedObjectReference> vms = networkMo.getVMsOnNetwork();
                  if (!CollectionUtils.isEmpty(vms)) {
                      logger.debug("Network port group: " + netDetails.getName() + " is in use");
                      return true;
                  }
              }
          }
  
          return false;
      }
  
      @Override
      public CopyVolumeAnswer execute(CopyVolumeCommand cmd) {
          try {
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              return (CopyVolumeAnswer) mgr.getStorageManager().execute(this, cmd);
          } catch (Throwable e) {
              return new CopyVolumeAnswer(cmd, false, createLogMessageException(e, cmd), null, null);
          }
      }
  
      @Override
      public void disconnected() {
      }
  
      @Override
      public IAgentControl getAgentControl() {
          return null;
      }
  
      @Override
      public PingCommand getCurrentStatus(long id) {
          try {
              gcAndKillHungWorkerVMs();
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              try {
                  if (!hyperHost.isHyperHostConnected()) {
                      return null;
                  }
              } catch (Exception e) {
                  logger.error("Unexpected exception", e);
                  return null;
              }
              return new PingRoutingCommand(getType(), id, syncHostVmStates());
          } finally {
              recycleServiceContext();
          }
      }
  
      private void gcAndKillHungWorkerVMs() {
          try {
              // take the chance to do left-over dummy VM cleanup from previous run
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              if (hyperHost.isHyperHostConnected()) {
                  mgr.gcLeftOverVMs(context);
  
                  logger.info("Scan hung worker VM to recycle");
  
                  int workerKey = ((HostMO) hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_WORKER);
                  int workerTagKey = ((HostMO) hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG);
                  String workerPropName = String.format("value[%d]", workerKey);
                  String workerTagPropName = String.format("value[%d]", workerTagKey);
  
                  // GC worker that has been running for too long
                  ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[]{"name", "config.template", workerPropName, workerTagPropName,});
                  if (ocs != null) {
                      for (ObjectContent oc : ocs) {
                          List<DynamicProperty> props = oc.getPropSet();
                          if (props != null) {
                              boolean template = false;
                              boolean isWorker = false;
                              String workerTag = null;
  
                              for (DynamicProperty prop : props) {
                                  if (prop.getName().equals("config.template")) {
                                      template = (Boolean) prop.getVal();
                                  } else if (prop.getName().equals(workerPropName)) {
                                      CustomFieldStringValue val = (CustomFieldStringValue) prop.getVal();
                                      if (val != null && val.getValue() != null && val.getValue().equalsIgnoreCase("true"))
                                          isWorker = true;
                                  } else if (prop.getName().equals(workerTagPropName)) {
                                      CustomFieldStringValue val = (CustomFieldStringValue) prop.getVal();
                                      workerTag = val.getValue();
                                  }
                              }
  
                              VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj());
                              if (!template && isWorker) {
                                  boolean recycle;
                                  recycle = mgr.needRecycle(workerTag);
  
                                  if (recycle) {
                                      logger.info("Recycle pending worker VM: " + vmMo.getName());
  
                                      vmMo.cancelPendingTasks();
                                      vmMo.powerOff();
                                      vmMo.detachAllDisksAndDestroy();
                                  }
                              }
                          }
                      }
                  }
              } else {
                  logger.error("Host is no longer connected.");
              }
  
          } catch (Throwable e) {
              if (e instanceof RemoteException) {
                  logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
                  invalidateServiceContext();
              }
          }
      }
  
      @Override
      public Type getType() {
          return com.cloud.host.Host.Type.Routing;
      }
  
      @Override
      public StartupCommand[] initialize() {
          return initialize(false);
      }
  
      @Override
      public StartupCommand[] initialize(boolean isTransferredConnection) {
          try {
              String hostApiVersion;
              VmwareContext context = getServiceContext();
              try {
                  VmwareHypervisorHost hyperHost = getHyperHost(context);
                  assert (hyperHost instanceof HostMO);
                  if (!((HostMO) hyperHost).isHyperHostConnected()) {
                      logger.info("Host " + hyperHost.getHyperHostName() + " is not in connected state");
                      return null;
                  }
  
                  ((HostMO) hyperHost).enableVncOnHostFirewall();
  
                  AboutInfo aboutInfo = ((HostMO) hyperHost).getHostAboutInfo();
                  hostApiVersion = aboutInfo.getApiVersion();
  
              } catch (Exception e) {
                  String msg = "VmwareResource intialize() failed due to : " + VmwareHelper.getExceptionMessage(e);
                  logger.error(msg);
                  invalidateServiceContext();
                  return null;
              }
  
              StartupRoutingCommand cmd = new StartupRoutingCommand();
              fillHostInfo(cmd);
              cmd.setHypervisorType(HypervisorType.VMware);
              cmd.setCluster(_cluster);
              cmd.setHypervisorVersion(hostApiVersion);
              cmd.setConnectionTransferred(isTransferredConnection);
  
              List<StartupStorageCommand> storageCmds = initializeLocalStorage();
              StartupCommand[] answerCmds = new StartupCommand[1 + storageCmds.size()];
              answerCmds[0] = cmd;
              for (int i = 0; i < storageCmds.size(); i++) {
                  answerCmds[i + 1] = storageCmds.get(i);
              }
  
              return answerCmds;
          } finally {
              recycleServiceContext();
          }
      }
  
      private List<StartupStorageCommand> initializeLocalStorage() {
          List<StartupStorageCommand> storageCmds = new ArrayList<>();
          VmwareContext context = getServiceContext();
  
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              if (hyperHost instanceof HostMO) {
                  HostMO hostMo = (HostMO) hyperHost;
  
                  List<Pair<ManagedObjectReference, String>> dsList = hostMo.getLocalDatastoreOnHost();
                  for (Pair<ManagedObjectReference, String> dsPair : dsList) {
                      DatastoreMO dsMo = new DatastoreMO(context, dsPair.first());
  
                      String poolUuid = dsMo.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID);
                      if (poolUuid == null || poolUuid.isEmpty()) {
                          poolUuid = UUID.randomUUID().toString();
                          dsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, poolUuid);
                      }
  
                      HypervisorHostHelper.createBaseFolder(dsMo, hyperHost, StoragePoolType.VMFS);
  
                      DatastoreSummary dsSummary = dsMo.getDatastoreSummary();
                      String address = hostMo.getHostName();
                      StoragePoolInfo pInfo = new StoragePoolInfo(poolUuid, address, dsMo.getMor().getValue(), "", StoragePoolType.VMFS, dsSummary.getCapacity(),
                              dsSummary.getFreeSpace());
                      StartupStorageCommand cmd = new StartupStorageCommand();
                      cmd.setName(poolUuid);
                      cmd.setPoolInfo(pInfo);
                      cmd.setGuid(poolUuid); // give storage host the same UUID as the local storage pool itself
                      cmd.setResourceType(Storage.StorageResourceType.STORAGE_POOL);
                      cmd.setDataCenter(_dcId);
                      cmd.setPod(_pod);
                      cmd.setCluster(_cluster);
  
                      logger.info("Add local storage startup command: " + _gson.toJson(cmd));
                      storageCmds.add(cmd);
                  }
  
              } else {
                  logger.info("Cluster host does not support local storage, skip it");
              }
          } catch (Exception e) {
              String msg = "initializing local storage failed due to : " + VmwareHelper.getExceptionMessage(e);
              logger.error(msg);
              invalidateServiceContext();
              throw new CloudRuntimeException(msg);
          }
  
          return storageCmds;
      }
  
      protected void fillHostInfo(StartupRoutingCommand cmd) {
          VmwareContext serviceContext = getServiceContext();
          Map<String, String> details = cmd.getHostDetails();
          if (details == null) {
              details = new HashMap<>();
          }
  
          try {
              fillHostHardwareInfo(serviceContext, cmd);
              fillHostNetworkInfo(serviceContext, cmd);
              fillHostDetailsInfo(serviceContext, details);
          } catch (RuntimeFaultFaultMsg e) {
              logger.error("RuntimeFault while retrieving host info: " + e, e);
              throw new CloudRuntimeException("RuntimeFault while retrieving host info");
          } catch (RemoteException e) {
              logger.error("RemoteException while retrieving host info: " + e, e);
              invalidateServiceContext();
              throw new CloudRuntimeException("RemoteException while retrieving host info");
          } catch (Exception e) {
              logger.error("Exception while retrieving host info: " + e, e);
              invalidateServiceContext();
              throw new CloudRuntimeException("Exception while retrieving host info: " + e);
          }
  
          cmd.setHostDetails(details);
          cmd.setName(_url);
          cmd.setGuid(_guid);
          cmd.setDataCenter(_dcId);
          cmd.setIqn(getIqn());
          cmd.setPod(_pod);
          cmd.setCluster(_cluster);
          cmd.setVersion(VmwareResource.class.getPackage().getImplementationVersion());
      }
  
      private String getIqn() {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
              if (hyperHost instanceof HostMO) {
                  HostMO host = (HostMO) hyperHost;
                  HostStorageSystemMO hostStorageSystem = host.getHostStorageSystemMO();
  
                  for (HostHostBusAdapter hba : hostStorageSystem.getStorageDeviceInfo().getHostBusAdapter()) {
                      if (hba instanceof HostInternetScsiHba) {
                          HostInternetScsiHba hostInternetScsiHba = (HostInternetScsiHba) hba;
  
                          if (hostInternetScsiHba.isIsSoftwareBased()) {
                              return ((HostInternetScsiHba) hba).getIScsiName();
                          }
                      }
                  }
              }
          } catch (Exception ex) {
              logger.info("Could not locate an IQN for this host.");
          }
  
          return null;
      }
  
      private void fillHostHardwareInfo(VmwareContext serviceContext, StartupRoutingCommand cmd) throws RuntimeFaultFaultMsg, RemoteException, Exception {
  
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          VmwareHypervisorHostResourceSummary summary = hyperHost.getHyperHostResourceSummary();
  
          if (logger.isInfoEnabled()) {
              logger.info("Startup report on host hardware info. " + _gson.toJson(summary));
          }
  
          cmd.setCaps("hvm");
          cmd.setDom0MinMemory(0);
          cmd.setSpeed(summary.getCpuSpeed());
          cmd.setCpuSockets(summary.getCpuSockets());
          cmd.setCpus((int) summary.getCpuCount());
          cmd.setMemory(summary.getMemoryBytes());
      }
  
      private void fillHostNetworkInfo(VmwareContext serviceContext, StartupRoutingCommand cmd) throws RuntimeFaultFaultMsg, RemoteException {
  
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
              assert (hyperHost instanceof HostMO);
              VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
  
              VmwareHypervisorHostNetworkSummary summary = hyperHost.getHyperHostNetworkSummary(mgr.getManagementPortGroupByHost((HostMO) hyperHost));
              if (summary == null) {
                  throw new Exception("No ESX(i) host found");
              }
  
              if (logger.isInfoEnabled()) {
                  logger.info("Startup report on host network info. " + _gson.toJson(summary));
              }
  
              cmd.setPrivateIpAddress(summary.getHostIp());
              cmd.setPrivateNetmask(summary.getHostNetmask());
              cmd.setPrivateMacAddress(summary.getHostMacAddress());
  
              cmd.setStorageIpAddress(summary.getHostIp());
              cmd.setStorageNetmask(summary.getHostNetmask());
              cmd.setStorageMacAddress(summary.getHostMacAddress());
  
          } catch (Throwable e) {
              String msg = "querying host network info failed due to " + VmwareHelper.getExceptionMessage(e);
              logger.error(msg, e);
              throw new CloudRuntimeException(msg);
          }
      }
  
      private void fillHostDetailsInfo(VmwareContext serviceContext, Map<String, String> details) throws Exception {
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
          if (hyperHost.isHAEnabled()) {
              details.put("NativeHA", "true");
          }
      }
  
      protected HashMap<String, HostVmStateReportEntry> syncHostVmStates() {
          try {
              return getHostVmStateReport();
          } catch (Exception e) {
              return new HashMap<>();
          }
      }
  
      protected OptionValue[] configureVnc(OptionValue[] optionsToMerge, VmwareHypervisorHost hyperHost, String vmName, String vncPassword, String keyboardLayout) throws Exception {
  
          VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
  
          VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
          if (!mgr.beginExclusiveOperation(600))
              throw new Exception("Unable to begin exclusive operation, lock time out");
  
          try {
              int maxVncPorts = 64;
              int vncPort = 0;
              Random random = new Random();
  
              HostMO vmOwnerHost = vmMo.getRunningHost();
  
              ManagedObjectReference morParent = vmOwnerHost.getParentMor();
              HashMap<String, Integer> portInfo;
              if (morParent.getType().equalsIgnoreCase("ClusterComputeResource")) {
                  ClusterMO clusterMo = new ClusterMO(vmOwnerHost.getContext(), morParent);
                  portInfo = clusterMo.getVmVncPortsOnCluster();
              } else {
                  portInfo = vmOwnerHost.getVmVncPortsOnHost();
              }
  
              // allocate first at 5900 - 5964 range
              Collection<Integer> existingPorts = portInfo.values();
              int val = random.nextInt(maxVncPorts);
              int startVal = val;
              do {
                  if (!existingPorts.contains(5900 + val)) {
                      vncPort = 5900 + val;
                      break;
                  }
  
                  val = (++val) % maxVncPorts;
              } while (val != startVal);
  
              if (vncPort == 0) {
                  logger.info("we've run out of range for ports between 5900-5964 for the cluster, we will try port range at 59000-60000");
  
                  Pair<Integer, Integer> additionalRange = mgr.getAddiionalVncPortRange();
                  maxVncPorts = additionalRange.second();
                  val = random.nextInt(maxVncPorts);
                  startVal = val;
                  do {
                      if (!existingPorts.contains(additionalRange.first() + val)) {
                          vncPort = additionalRange.first() + val;
                          break;
                      }
  
                      val = (++val) % maxVncPorts;
                  } while (val != startVal);
              }
  
              if (vncPort == 0) {
                  throw new Exception("Unable to find an available VNC port on host");
              }
  
              if (logger.isInfoEnabled()) {
                  logger.info("Configure VNC port for VM " + vmName + ", port: " + vncPort + ", host: " + vmOwnerHost.getHyperHostName());
              }
  
              return VmwareHelper.composeVncOptions(optionsToMerge, true, vncPassword, vncPort, keyboardLayout);
          } finally {
              try {
                  mgr.endExclusiveOperation();
              } catch (Throwable e) {
                  assert (false);
                  logger.error("Unexpected exception ", e);
              }
          }
      }
  
      private VirtualMachineGuestOsIdentifier translateGuestOsIdentifier(String cpuArchitecture, String guestOs, String cloudGuestOs) {
          if (cpuArchitecture == null) {
              logger.warn("CPU arch is not set, default to i386. guest os: " + guestOs);
              cpuArchitecture = "i386";
          }
  
          if (cloudGuestOs == null) {
              logger.warn("Guest OS mapping name is not set for guest os: " + guestOs);
          }
  
          VirtualMachineGuestOsIdentifier identifier = null;
          try {
              if (cloudGuestOs != null) {
                  identifier = VirtualMachineGuestOsIdentifier.fromValue(cloudGuestOs);
                  logger.debug("Using mapping name : " + identifier.toString());
              }
          } catch (IllegalArgumentException e) {
              logger.warn("Unable to find Guest OS Identifier in VMware for mapping name: " + cloudGuestOs + ". Continuing with defaults.");
          }
          if (identifier != null) {
              return identifier;
          }
  
          if (cpuArchitecture.equalsIgnoreCase("x86_64")) {
              return VirtualMachineGuestOsIdentifier.OTHER_GUEST_64;
          }
          return VirtualMachineGuestOsIdentifier.OTHER_GUEST;
      }
  
      private HashMap<String, HostVmStateReportEntry> getHostVmStateReport() throws Exception {
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
          int key = ((HostMO) hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
          if (key == 0) {
              logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!");
          }
          String instanceNameCustomField = "value[" + key + "]";
  
          // CLOUD_VM_INTERNAL_NAME stores the internal CS generated vm name. This was earlier stored in name. Now, name can be either the hostname or
          // the internal CS name, but the custom field CLOUD_VM_INTERNAL_NAME always stores the internal CS name.
          ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[]{"name", "runtime.powerState", "config.template", instanceNameCustomField});
  
          HashMap<String, HostVmStateReportEntry> newStates = new HashMap<>();
          if (ocs != null && ocs.length > 0) {
              for (ObjectContent oc : ocs) {
                  List<DynamicProperty> objProps = oc.getPropSet();
                  if (objProps != null) {
  
                      boolean isTemplate = false;
                      String name = null;
                      String VMInternalCSName = null;
                      VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF;
                      for (DynamicProperty objProp : objProps) {
                          if (objProp.getName().equals("config.template")) {
                              if (objProp.getVal().toString().equalsIgnoreCase("true")) {
                                  isTemplate = true;
                              }
                          } else if (objProp.getName().equals("runtime.powerState")) {
                              powerState = (VirtualMachinePowerState) objProp.getVal();
                          } else if (objProp.getName().equals("name")) {
                              name = (String) objProp.getVal();
                          } else if (objProp.getName().contains(instanceNameCustomField)) {
                              if (objProp.getVal() != null)
                                  VMInternalCSName = ((CustomFieldStringValue) objProp.getVal()).getValue();
                          } else {
                              assert (false);
                          }
                      }
  
                      if (VMInternalCSName != null)
                          name = VMInternalCSName;
  
                      if (!isTemplate) {
                          newStates.put(name, new HostVmStateReportEntry(convertPowerState(powerState), hyperHost.getHyperHostName()));
                      }
                  }
              }
          }
          return newStates;
      }
  
      private HashMap<String, PowerState> getVmStates() throws Exception {
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
          int key = ((HostMO) hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
          if (key == 0) {
              logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!");
          }
          String instanceNameCustomField = "value[" + key + "]";
  
          // CLOUD_VM_INTERNAL_NAME stores the internal CS generated vm name. This was earlier stored in name. Now, name can be either the hostname or
          // the internal CS name, but the custom field CLOUD_VM_INTERNAL_NAME always stores the internal CS name.
          ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[]{"name", "runtime.powerState", "config.template", instanceNameCustomField});
  
          HashMap<String, PowerState> newStates = new HashMap<>();
          if (ocs != null && ocs.length > 0) {
              for (ObjectContent oc : ocs) {
                  List<DynamicProperty> objProps = oc.getPropSet();
                  if (objProps != null) {
  
                      boolean isTemplate = false;
                      String name = null;
                      String VMInternalCSName = null;
                      VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF;
                      for (DynamicProperty objProp : objProps) {
                          if (objProp.getName().equals("config.template")) {
                              if (objProp.getVal().toString().equalsIgnoreCase("true")) {
                                  isTemplate = true;
                              }
                          } else if (objProp.getName().equals("runtime.powerState")) {
                              powerState = (VirtualMachinePowerState) objProp.getVal();
                          } else if (objProp.getName().equals("name")) {
                              name = (String) objProp.getVal();
                          } else if (objProp.getName().contains(instanceNameCustomField)) {
                              if (objProp.getVal() != null)
                                  VMInternalCSName = ((CustomFieldStringValue) objProp.getVal()).getValue();
                          } else {
                              assert (false);
                          }
                      }
  
                      if (VMInternalCSName != null)
                          name = VMInternalCSName;
  
                      if (!isTemplate) {
                          newStates.put(name, convertPowerState(powerState));
                      }
                  }
              }
          }
          return newStates;
      }
  
      private HashMap<String, VmStatsEntry> getVmStats(List<String> vmNames) throws Exception {
          VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
          HashMap<String, VmStatsEntry> vmResponseMap = new HashMap<>();
          ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager();
          VimPortType service = getServiceContext().getService();
  
          PerfCounterInfo rxPerfCounterInfo = null;
          PerfCounterInfo txPerfCounterInfo = null;
          PerfCounterInfo diskReadIOPerfCounterInfo = null;
          PerfCounterInfo diskWriteIOPerfCounterInfo = null;
          PerfCounterInfo diskReadKbsPerfCounterInfo = null;
          PerfCounterInfo diskWriteKbsPerfCounterInfo = null;
  
          Integer windowInterval = getVmwareWindowTimeInterval();
          final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), windowInterval);
          final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0);
  
          List<PerfCounterInfo> cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter");
          for (PerfCounterInfo info : cInfo) {
              if ("net".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) {
                  if ("transmitted".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      txPerfCounterInfo = info;
                  }
                  if ("received".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      rxPerfCounterInfo = info;
                  }
              }
              if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey())) {
                  if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      diskReadIOPerfCounterInfo = info;
                  }
                  if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      diskWriteIOPerfCounterInfo = info;
                  }
                  if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      diskReadKbsPerfCounterInfo = info;
                  }
                  if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) {
                      diskWriteKbsPerfCounterInfo = info;
                  }
              }
          }
  
          int key = ((HostMO) hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
          if (key == 0) {
              logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!");
          }
          String instanceNameCustomField = "value[" + key + "]";
  
          final String numCpuStr = "summary.config.numCpu";
          final String cpuUseStr = "summary.quickStats.overallCpuUsage";
          final String guestMemUseStr = "summary.quickStats.guestMemoryUsage";
          final String memLimitStr = "resourceConfig.memoryAllocation.limit";
          final String memMbStr = "config.hardware.memoryMB";
          final String allocatedCpuStr = "summary.runtime.maxCpuUsage";
  
          ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[]{
                  "name", numCpuStr, cpuUseStr, guestMemUseStr, memLimitStr, memMbStr, allocatedCpuStr, instanceNameCustomField
          });
  
          if (ocs != null && ocs.length > 0) {
              for (ObjectContent oc : ocs) {
                  List<DynamicProperty> objProps = oc.getPropSet();
                  if (objProps != null) {
                      String name;
                      String numberCPUs = null;
                      double maxCpuUsage = 0;
                      String memlimit = null;
                      String memkb = null;
                      String guestMemusage = null;
                      String vmNameOnVcenter = null;
                      String vmInternalCSName = null;
                      double allocatedCpu = 0;
                      for (DynamicProperty objProp : objProps) {
                          if (objProp.getName().equals("name")) {
                              vmNameOnVcenter = objProp.getVal().toString();
                          } else if (objProp.getName().contains(instanceNameCustomField)) {
                              if (objProp.getVal() != null)
                                  vmInternalCSName = ((CustomFieldStringValue) objProp.getVal()).getValue();
                          } else if (objProp.getName().equals(guestMemUseStr)) {
                              guestMemusage = objProp.getVal().toString();
                          } else if (objProp.getName().equals(numCpuStr)) {
                              numberCPUs = objProp.getVal().toString();
                          } else if (objProp.getName().equals(cpuUseStr)) {
                              maxCpuUsage = NumberUtils.toDouble(objProp.getVal().toString());
                          } else if (objProp.getName().equals(memLimitStr)) {
                              memlimit = objProp.getVal().toString();
                          } else if (objProp.getName().equals(memMbStr)) {
                              memkb = objProp.getVal().toString();
                          } else if (objProp.getName().equals(allocatedCpuStr)) {
                              allocatedCpu = NumberUtils.toDouble(objProp.getVal().toString());
                          }
                      }
  
                      maxCpuUsage = (maxCpuUsage / allocatedCpu) * 100;
                      if (vmInternalCSName != null) {
                          name = vmInternalCSName;
                      } else {
                          name = vmNameOnVcenter;
                      }
  
                      if (!vmNames.contains(name)) {
                          continue;
                      }
  
                      ManagedObjectReference vmMor = hyperHost.findVmOnHyperHost(name).getMor();
                      assert (vmMor != null);
  
                      double networkReadKBs = 0;
                      double networkWriteKBs = 0;
                      double diskReadIops = 0;
                      double diskWriteIops = 0;
                      double diskReadKbs = 0;
                      double diskWriteKbs = 0;
  
                      final ArrayList<PerfMetricId> perfMetricsIds = new ArrayList<>();
                      if (rxPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(rxPerfCounterInfo, ""));
                      }
                      if (txPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(txPerfCounterInfo, ""));
                      }
                      if (diskReadIOPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, "*"));
                      }
                      if (diskWriteIOPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, "*"));
                      }
                      if (diskReadKbsPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, ""));
                      }
                      if (diskWriteKbsPerfCounterInfo != null) {
                          perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, ""));
                      }
  
                      if (perfMetricsIds.size() > 0) {
                          try {
                              final PerfQuerySpec qSpec = new PerfQuerySpec();
                              qSpec.setEntity(vmMor);
                              qSpec.setFormat("normal");
                              qSpec.setIntervalId(windowInterval);
                              qSpec.setStartTime(startTime);
                              qSpec.setEndTime(endTime);
                              qSpec.getMetricId().addAll(perfMetricsIds);
                              final List<PerfEntityMetricBase> perfValues = service.queryPerf(perfMgr, Collections.singletonList(qSpec));
                              for (final PerfEntityMetricBase perfValue : perfValues) {
                                  if (!(perfValue instanceof PerfEntityMetric)) {
                                      continue;
                                  }
                                  final List<PerfMetricSeries> seriesList = ((PerfEntityMetric) perfValue).getValue();
                                  for (final PerfMetricSeries series : seriesList) {
                                      if (!(series instanceof PerfMetricIntSeries)) {
                                          continue;
                                      }
                                      final List<Long> values = ((PerfMetricIntSeries) series).getValue();
                                      double sum = 0;
                                      for (final Long value : values) {
                                          sum += value;
                                      }
                                      double avg = sum / (values.size() * 1f);
                                      if (series.getId().getCounterId() == rxPerfCounterInfo.getKey()) {
                                          networkReadKBs = avg;
                                      }
                                      if (series.getId().getCounterId() == txPerfCounterInfo.getKey()) {
                                          networkWriteKBs = avg;
                                      }
                                      if (series.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) {
                                          diskReadIops += avg;
                                      }
                                      if (series.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) {
                                          diskWriteIops += avg;
                                      }
                                      if (series.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) {
                                          diskReadKbs = avg;
                                      }
                                      if (series.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) {
                                          diskWriteKbs = avg;
                                      }
                                  }
                              }
                          } catch (Exception e) {
                              logger.error(String.format("Unable to execute PerfQuerySpec due to: [%s]. The window interval is enabled in vCenter?", VmwareHelper.getExceptionMessage(e)), e);
                          }
                      }
  
                      double doubleMemKb = NumberUtils.toDouble(memkb);
                      double guestFreeMem =  doubleMemKb - NumberUtils.toDouble(guestMemusage);
  
                      final VmStatsEntry vmStats = new VmStatsEntry(0, doubleMemKb * 1024, guestFreeMem * 1024, NumberUtils.toDouble(memlimit) * 1024, maxCpuUsage, networkReadKBs,
                              networkWriteKBs, NumberUtils.toInt(numberCPUs), diskReadKbs, diskWriteKbs, diskReadIops, diskWriteIops, "vm");
                      vmResponseMap.put(name, vmStats);
  
                  }
              }
          }
          return vmResponseMap;
      }
  
      protected String networkUsage(final String privateIpAddress, final String option, final String ethName) {
          return networkUsage(privateIpAddress, option, ethName, null);
      }
  
      protected String networkUsage(final String privateIpAddress, final String option, final String ethName, final String publicIp) {
          String args = null;
          if (option.equals("get")) {
              args = "-g";
              if (StringUtils.isNotEmpty(publicIp)) {
                  args += " -l " + publicIp;
              }
          } else if (option.equals("create")) {
              args = "-c";
          } else if (option.equals("reset")) {
              args = "-r";
          } else if (option.equals("addVif")) {
              args = "-a";
              args += ethName;
          } else if (option.equals("deleteVif")) {
              args = "-d";
              args += ethName;
          }
  
          ExecutionResult result = executeInVR(privateIpAddress, "netusage.sh", args);
  
          if (!result.isSuccess()) {
              return null;
          }
  
          return result.getDetails();
      }
  
      protected long[] getNetworkStats(String privateIP, String publicIp) {
          String result = networkUsage(privateIP, "get", null, publicIp);
          long[] stats = new long[2];
          if (result != null) {
              try {
                  String[] splitResult = result.split(":");
                  int i = 0;
                  while (i < splitResult.length - 1) {
                      stats[0] += Long.parseLong(splitResult[i++]);
                      stats[1] += Long.parseLong(splitResult[i++]);
                  }
              } catch (Throwable e) {
                  logger.warn("Unable to parse return from script return of network usage command: " + e, e);
              }
          }
          return stats;
      }
  
      protected String connect(final String vmName, final String ipAddress, final int port) {
          long startTick = System.currentTimeMillis();
  
          // wait until we have at least been waiting for _ops_timeout time or
          // at least have tried _retry times, this is to coordinate with system
          // VM patching/rebooting time that may need
          int retry = _retry;
          while (System.currentTimeMillis() - startTick <= _opsTimeout || --retry > 0) {
              logger.info("Trying to connect to " + ipAddress);
              try (SocketChannel sch = SocketChannel.open();) {
                  sch.configureBlocking(true);
                  sch.socket().setSoTimeout(5000);
  
                  InetSocketAddress addr = new InetSocketAddress(ipAddress, port);
                  sch.connect(addr);
                  return null;
              } catch (IOException e) {
                  logger.info("Could not connect to " + ipAddress + " due to " + e);
                  if (e instanceof ConnectException) {
                      // if connection is refused because of VM is being started,
                      // we give it more sleep time
                      // to avoid running out of retry quota too quickly
                      try {
                          Thread.sleep(5000);
                      } catch (InterruptedException ex) {
                          logger.debug("[ignored] interrupted while waiting to retry connect after failure.", e);
                      }
                  }
              }
  
              try {
                  Thread.sleep(1000);
              } catch (InterruptedException ex) {
                  logger.debug("[ignored] interrupted while waiting to retry connect.");
              }
          }
  
          logger.info("Unable to logon to " + ipAddress);
  
          return "Unable to connect";
      }
  
      protected String connect(final String vmname, final String ipAddress) {
          return connect(vmname, ipAddress, 3922);
      }
  
      public static PowerState getVmState(VirtualMachineMO vmMo) throws Exception {
          VirtualMachineRuntimeInfo runtimeInfo = vmMo.getRuntimeInfo();
          return convertPowerState(runtimeInfo.getPowerState());
      }
  
      private static PowerState convertPowerState(VirtualMachinePowerState powerState) {
          return s_powerStatesTable.get(powerState);
      }
  
      public static PowerState getVmPowerState(VirtualMachineMO vmMo) throws Exception {
          VirtualMachineRuntimeInfo runtimeInfo = vmMo.getRuntimeInfo();
          return convertPowerState(runtimeInfo.getPowerState());
      }
  
      private static HostStatsEntry getHyperHostStats(VmwareHypervisorHost hyperHost) throws Exception {
          ComputeResourceSummary hardwareSummary = hyperHost.getHyperHostHardwareSummary();
          if (hardwareSummary == null)
              return null;
  
          HostStatsEntry entry = new HostStatsEntry();
  
          entry.setEntityType("host");
          double cpuUtilization = ((double) (hardwareSummary.getTotalCpu() - hardwareSummary.getEffectiveCpu()) / (double) hardwareSummary.getTotalCpu() * 100);
          entry.setCpuUtilization(cpuUtilization);
          entry.setTotalMemoryKBs(hardwareSummary.getTotalMemory() / 1024);
          entry.setFreeMemoryKBs(hardwareSummary.getEffectiveMemory() * 1024);
  
          return entry;
      }
  
      private String getRouterSshControlIp(NetworkElementCommand cmd) {
          String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
          if (logger.isDebugEnabled())
              logger.debug("Use router's private IP for SSH control. IP : " + routerIp);
          return routerIp;
      }
  
      @Override
      public void setAgentControl(IAgentControl agentControl) {
      }
  
      @Override
      public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
          try {
              _name = name;
  
              _url = (String) params.get("url");
              _username = (String) params.get("username");
              _password = (String) params.get("password");
              _dcId = (String) params.get("zone");
              _pod = (String) params.get("pod");
              _cluster = (String) params.get("cluster");
  
              _guid = (String) params.get("guid");
              String[] tokens = _guid.split("@");
              _vCenterAddress = tokens[1];
              _morHyperHost = new ManagedObjectReference();
              String[] hostTokens = tokens[0].split(":");
              _morHyperHost.setType(hostTokens[0]);
              _morHyperHost.setValue(hostTokens[1]);
  
              _guestTrafficInfo = (VmwareTrafficLabel) params.get("guestTrafficInfo");
              _publicTrafficInfo = (VmwareTrafficLabel) params.get("publicTrafficInfo");
              VmwareContext context = getServiceContext();
              VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
              if (mgr == null) {
                  throw new ConfigurationException("Invalid vmwareContext:  vmwareMgr stock object is not set or cleared.");
              }
              mgr.setupResourceStartupParams(params);
  
              CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager());
              cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID);
              cfmMo.ensureCustomFieldDef("StoragePod", CustomFieldConstants.CLOUD_UUID);
  
              if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch
                      || _guestTrafficInfo != null && _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) {
                  cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP);
              }
              cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC);
              cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
              cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK);
              cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
              cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER);
              cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG);
  
              VmwareHypervisorHost hostMo = this.getHyperHost(context);
              _hostName = hostMo.getHyperHostName();
  
              if (_guestTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch
                      || _publicTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) {
                  _privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
                  _vsmCredentials = mgr.getNexusVSMCredentialsByClusterId(Long.parseLong(_cluster));
              }
  
              if (_privateNetworkVSwitchName == null) {
                  _privateNetworkVSwitchName = (String) params.get("private.network.vswitch.name");
              }
  
              String value = (String) params.get("vmware.recycle.hung.wokervm");
              if (value != null && value.equalsIgnoreCase("true"))
                  _recycleHungWorker = true;
  
              value = (String) params.get("vmware.root.disk.controller");
              if (value != null && value.equalsIgnoreCase("scsi"))
                  _rootDiskController = DiskControllerType.scsi;
              else if (value != null && value.equalsIgnoreCase("ide"))
                  _rootDiskController = DiskControllerType.ide;
              else
                  _rootDiskController = DiskControllerType.osdefault;
  
              Integer intObj = (Integer) params.get("ports.per.dvportgroup");
              if (intObj != null)
                  _portsPerDvPortGroup = intObj;
  
              logger.info("VmwareResource network configuration info." + " private traffic over vSwitch: " + _privateNetworkVSwitchName + ", public traffic over "
                      + _publicTrafficInfo.getVirtualSwitchType() + " : " + _publicTrafficInfo.getVirtualSwitchName() + ", guest traffic over "
                      + _guestTrafficInfo.getVirtualSwitchType() + " : " + _guestTrafficInfo.getVirtualSwitchName());
  
              Boolean boolObj = (Boolean) params.get("vmware.create.full.clone");
              if (boolObj != null && boolObj) {
                  _fullCloneFlag = true;
              } else {
                  _fullCloneFlag = false;
              }
  
              boolObj = (Boolean) params.get("vm.instancename.flag");
              if (boolObj != null && boolObj) {
                  _instanceNameFlag = true;
              } else {
                  _instanceNameFlag = false;
              }
  
              value = (String) params.get("scripts.timeout");
              int timeout = NumbersUtil.parseInt(value, 1440) * 1000;
  
              storageNfsVersion = NfsSecondaryStorageResource.retrieveNfsVersionFromParams(params);
              _storageProcessor = new VmwareStorageProcessor((VmwareHostService) this, _fullCloneFlag, (VmwareStorageMount) mgr, timeout, this, _shutdownWaitMs, null,
                      storageNfsVersion);
              storageHandler = new VmwareStorageSubsystemCommandHandler(_storageProcessor, storageNfsVersion);
  
              _vrResource = new VirtualRoutingResource(this);
              if (!_vrResource.configure(name, params)) {
                  throw new ConfigurationException("Unable to configure VirtualRoutingResource");
              }
  
              if (logger.isTraceEnabled()) {
                  logger.trace("Successfully configured VmwareResource.");
              }
              return true;
          } catch (Exception e) {
              logger.error("Unexpected Exception ", e);
              throw new ConfigurationException("Failed to configure VmwareResource due to unexpect exception.");
          } finally {
              recycleServiceContext();
          }
      }
  
      @Override
      public String getName() {
          return _name;
      }
  
      @Override
      protected String getDefaultScriptsDir() {
          return null;
      }
  
      @Override
      public boolean start() {
          return true;
      }
  
      @Override
      public boolean stop() {
          return true;
      }
  
      public VmwareContext getServiceContext() {
          return getServiceContext(null);
      }
  
      public void invalidateServiceContext() {
          invalidateServiceContext(null);
      }
  
      public VmwareHypervisorHost getHyperHost(VmwareContext context) {
          return getHyperHost(context, null);
      }
  
      @Override
      public VmwareContext getServiceContext(Command cmd) {
          VmwareContext context;
          if (s_serviceContext.get() != null) {
              context = s_serviceContext.get();
              String poolKey = VmwareContextPool.composePoolKey(_vCenterAddress, _username);
              // Before re-using the thread local context, ensure it corresponds to the right vCenter API session and that it is valid to make calls.
              if (context.getPoolKey().equals(poolKey)) {
                  if (context.validate()) {
                      if (logger.isTraceEnabled()) {
                          logger.trace("ThreadLocal context is still valid, just reuse");
                      }
                      return context;
                  } else {
                      logger.info("Validation of the context failed, dispose and use a new one");
                      invalidateServiceContext(context);
                  }
              } else {
                  // Exisitng ThreadLocal context corresponds to a different vCenter API session. Why has it not been recycled?
                  logger.warn("ThreadLocal VMware context: " + poolKey + " doesn't correspond to the right vCenter. Expected VMware context: " + context.getPoolKey());
              }
          }
          try {
              context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password);
              s_serviceContext.set(context);
          } catch (Exception e) {
              logger.error("Unable to connect to vSphere server: " + _vCenterAddress, e);
              throw new CloudRuntimeException("Unable to connect to vSphere server: " + _vCenterAddress);
          }
          return context;
      }
  
      @Override
      public void invalidateServiceContext(VmwareContext context) {
          assert (s_serviceContext.get() == context);
  
          s_serviceContext.set(null);
          if (context != null)
              context.close();
      }
  
      private void recycleServiceContext() {
          VmwareContext context = s_serviceContext.get();
          if (logger.isTraceEnabled()) {
              logger.trace("Reset threadlocal context to null");
          }
          s_serviceContext.set(null);
  
          if (context != null) {
              assert (context.getPool() != null);
              if (logger.isTraceEnabled()) {
                  logger.trace("Recycling threadlocal context to pool");
              }
              context.getPool().registerContext(context);
          }
      }
  
      @Override
      public VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd) {
          if (_morHyperHost.getType().equalsIgnoreCase("HostSystem")) {
              return new HostMO(context, _morHyperHost);
          }
          return new ClusterMO(context, _morHyperHost);
      }
  
      @Override
      @DB
      public String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO sourceDsMo) throws Exception {
          VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
          String vmName = mgr.composeWorkerName();
          if (sourceDsMo!= null && sourceDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
              vmName = CustomFieldConstants.CLOUD_UUID + "-" + vmName;
          }
  
          assert (cmd != null);
          context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
          // TODO: Fix this? long checkPointId = vmwareMgr.pushCleanupCheckpoint(this._guid, vmName);
          // TODO: Fix this? cmd.setContextParam("checkpoint", String.valueOf(checkPointId));
          return vmName;
      }
  
      @Override
      public void setName(String name) {
          // TODO Auto-generated method stub
      }
  
      @Override
      public void setConfigParams(Map<String, Object> params) {
          // TODO Auto-generated method stub
  
      }
  
      @Override
      public Map<String, Object> getConfigParams() {
          // TODO Auto-generated method stub
          return null;
      }
  
      @Override
      public int getRunLevel() {
          // TODO Auto-generated method stub
          return 0;
      }
  
      @Override
      public void setRunLevel(int level) {
          // TODO Auto-generated method stub
      }
  
      @Override
      public Answer execute(DestroyCommand cmd) {
          try {
              VmwareContext context = getServiceContext(null);
              VmwareHypervisorHost hyperHost = getHyperHost(context, null);
              VolumeTO vol = cmd.getVolume();
  
              VirtualMachineMO vmMo = findVmOnDatacenter(context, hyperHost, vol);
  
              if (vmMo != null) {
                  if (logger.isInfoEnabled()) {
                      logger.info("Destroy template volume " + vol.getPath());
                  }
                  if (vmMo.isTemplate()) {
                      vmMo.markAsVirtualMachine(hyperHost.getHyperHostOwnerResourcePool(), hyperHost.getMor());
                  }
                  vmMo.destroy();
              } else {
                  if (logger.isInfoEnabled()) {
                      logger.info("Template volume " + vol.getPath() + " is not found, no need to delete.");
                  }
              }
              return new Answer(cmd, true, "Success");
  
          } catch (Throwable e) {
              return new Answer(cmd, false, createLogMessageException(e, cmd));
          }
      }
  
      /**
       * Use data center to look for vm, instead of randomly picking up a cluster<br/>
       * (in multiple cluster environments vm could not be found if wrong cluster was chosen)
       *
       * @param context   vmware context
       * @param hyperHost vmware hv host
       * @param vol       volume
       * @return a virtualmachinemo if could be found on datacenter.
       * @throws Exception             if there is an error while finding vm
       * @throws CloudRuntimeException if datacenter cannot be found
       */
      protected VirtualMachineMO findVmOnDatacenter(VmwareContext context, VmwareHypervisorHost hyperHost, VolumeTO vol) throws Exception {
          DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter());
          if (dcMo.getMor() == null) {
              String msg = "Unable to find VMware DC";
              logger.error(msg);
              throw new CloudRuntimeException(msg);
          }
          return dcMo.findVm(vol.getPath());
      }
  
      protected File getSystemVmKeyFile() {
          if (s_systemVmKeyFile == null) {
              syncFetchSystemVmKeyFile();
          }
          return s_systemVmKeyFile;
      }
  
      private void syncFetchSystemVmKeyFile() {
          synchronized (s_syncLockObjectFetchKeyFile) {
              if (s_systemVmKeyFile == null) {
                  s_systemVmKeyFile = fetchSystemVmKeyFile();
              }
          }
      }
  
      private File fetchSystemVmKeyFile() {
          String filePath = s_relativePathSystemVmKeyFileInstallDir;
          logger.debug("Looking for file [" + filePath + "] in the classpath.");
          URL url = Script.class.getClassLoader().getResource(filePath);
          File keyFile = null;
          if (url != null) {
              keyFile = new File(url.getPath());
          }
          if (keyFile == null || !keyFile.exists()) {
              filePath = s_defaultPathSystemVmKeyFile;
              keyFile = new File(filePath);
              logger.debug("Looking for file [" + filePath + "] in the classpath.");
          }
          if (!keyFile.exists()) {
              logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile);
          }
          return keyFile;
      }
  
      private Answer execute(GetUnmanagedInstancesCommand cmd) {
          VmwareContext context = getServiceContext();
          HashMap<String, UnmanagedInstanceTO> unmanagedInstances = new HashMap<>();
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(context);
  
              String vmName = cmd.getInstanceName();
              List<VirtualMachineMO> vmMos = hyperHost.listVmsOnHyperHostWithHypervisorName(vmName);
  
              for (VirtualMachineMO vmMo : vmMos) {
                  if (vmMo == null) {
                      continue;
                  }
                  if (vmMo.isTemplate()) {
                      continue;
                  }
                  // Filter managed instances
                  if (cmd.hasManagedInstance(vmMo.getName()) ||
                          (StringUtils.isNotEmpty(vmMo.getInternalCSName())
                                  && Objects.equals(vmMo.getName(), vmMo.getInternalCSName())
                                  && cmd.hasManagedInstance(vmMo.getInternalCSName()))) {
                      continue;
                  }
                  // Filter instance if answer is requested for a particular instance name
                  if (StringUtils.isNotEmpty(cmd.getInstanceName()) &&
                          !cmd.getInstanceName().equals(vmMo.getVmName())) {
                      continue;
                  }
                  UnmanagedInstanceTO instance = VmwareHelper.getUnmanagedInstance(hyperHost, vmMo);
                  if (instance != null) {
                      VirtualMachineConfigSummary configSummary = vmMo.getConfigSummary();
                      instance.setCpuSpeed(configSummary != null ? configSummary.getCpuReservation() : 0);
                      unmanagedInstances.put(instance.getName(), instance);
                  }
              }
          } catch (Exception e) {
              logger.info("GetUnmanagedInstancesCommand failed due to " + VmwareHelper.getExceptionMessage(e));
          }
          return new GetUnmanagedInstancesAnswer(cmd, "", unmanagedInstances);
      }
  
      private Answer execute(PrepareUnmanageVMInstanceCommand cmd) {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          String instanceName = cmd.getInstanceName();
  
          try {
              logger.debug(String.format("Verify if VMware instance: [%s] is available before unmanaging VM.", cmd.getInstanceName()));
  
              ManagedObjectReference  dcMor = hyperHost.getHyperHostDatacenter();
              DatacenterMO dataCenterMo = new DatacenterMO(getServiceContext(), dcMor);
              VirtualMachineMO vm = dataCenterMo.findVm(instanceName);
              if (vm == null) {
                  return new PrepareUnmanageVMInstanceAnswer(cmd, false, String.format("Cannot find VM with name [%s] in datacenter [%s].", instanceName, dataCenterMo.getName()));
              }
          } catch (Exception e) {
              logger.error("Error trying to verify if VM to unmanage exists", e);
              return new PrepareUnmanageVMInstanceAnswer(cmd, false, "Error: " + e.getMessage());
          }
  
          return new PrepareUnmanageVMInstanceAnswer(cmd, true, "OK");
      }
  
      /*
       * Method to relocate a virtual machine. This migrates VM and its volumes to given host, datastores.
       * It is used for MigrateVolumeCommand (detached volume case), MigrateVmToPoolCommand and MigrateVmWithStorageCommand.
       */
  
      private List<VolumeObjectTO> relocateVirtualMachine(final VmwareHypervisorHost hypervisorHost,
                                                          final String name, final VirtualMachineTO vmTo,
                                                          final String targetHost, final VmwareHypervisorHost hostInTargetCluster,
                                                          final String poolUuid, final List<Pair<VolumeTO, StorageFilerTO>> volToFiler) throws Exception {
          String vmName = name;
          if (vmName == null && vmTo != null) {
              vmName = vmTo.getName();
          }
          VmwareHypervisorHost sourceHyperHost = hypervisorHost;
          VmwareHypervisorHost targetHyperHost = hostInTargetCluster;
          VirtualMachineMO vmMo;
          ManagedObjectReference morSourceHostDc;
          VirtualMachineRelocateSpec relocateSpec = new VirtualMachineRelocateSpec();
          List<VirtualMachineRelocateSpecDiskLocator> diskLocators = new ArrayList<>();
          Set<String> mountedDatastoresAtSource = new HashSet<>();
          List<VolumeObjectTO> volumeToList =  new ArrayList<>();
          Map<Long, Integer> volumeDeviceKey = new HashMap<>();
  
          try {
              if (sourceHyperHost == null) {
                  sourceHyperHost = getHyperHost(getServiceContext());
              }
              if (targetHyperHost == null && StringUtils.isNotBlank(targetHost)) {
                  targetHyperHost = VmwareHelper.getHostMOFromHostName(getServiceContext(), targetHost);
              }
              morSourceHostDc = sourceHyperHost.getHyperHostDatacenter();
              DatacenterMO dcMo = new DatacenterMO(sourceHyperHost.getContext(), morSourceHostDc);
              if (targetHyperHost != null) {
                  ManagedObjectReference morTargetHostDc = targetHyperHost.getHyperHostDatacenter();
                  if (!morSourceHostDc.getValue().equalsIgnoreCase(morTargetHostDc.getValue())) {
                      String msg = String.format("VM: %s cannot be migrated between different datacenter", vmName);
                      throw new CloudRuntimeException(msg);
                  }
              }
  
              // find VM through source host (VM is not at the target host yet)
              vmMo = sourceHyperHost.findVmOnHyperHost(vmName);
              if (vmMo == null) {
                  String msg = String.format("VM: %s does not exist on host: %s", vmName, sourceHyperHost.getHyperHostName());
                  logger.warn(msg);
                  // find VM through source host (VM is not at the target host yet)
                  vmMo = dcMo.findVm(vmName);
                  if (vmMo == null) {
                      msg = String.format("VM: %s does not exist on datacenter: %s", vmName, dcMo.getName());
                      logger.error(msg);
                      throw new Exception(msg);
                  }
                  // VM host has changed
                  sourceHyperHost = vmMo.getRunningHost();
              }
  
              vmName = vmMo.getName();
              String srcHostApiVersion = ((HostMO)sourceHyperHost).getHostAboutInfo().getApiVersion();
  
              if (StringUtils.isNotBlank(poolUuid)) {
                  VmwareHypervisorHost dsHost = targetHyperHost == null ? sourceHyperHost : targetHyperHost;
                  ManagedObjectReference morDatastore;
                  morDatastore = getTargetDatastoreMOReference(poolUuid, dsHost);
                  if (morDatastore == null) {
                      String msg = String.format("Unable to find the target datastore: %s on host: %s to execute migration", poolUuid, dsHost.getHyperHostName());
                      logger.error(msg);
                      throw new CloudRuntimeException(msg);
                  }
                  relocateSpec.setDatastore(morDatastore);
              } else if (CollectionUtils.isNotEmpty(volToFiler)) {
                  // Specify destination datastore location for each volume
                  VmwareHypervisorHost dsHost = targetHyperHost == null ? sourceHyperHost : targetHyperHost;
                  for (Pair<VolumeTO, StorageFilerTO> entry : volToFiler) {
                      VolumeTO volume = entry.first();
                      StorageFilerTO filerTo = entry.second();
                      if (logger.isDebugEnabled()) {
                          logger.debug(String.format("Preparing spec for volume: %s to migrate it to datastore: %s", volume.getName(), filerTo.getUuid()));
                      }
                      ManagedObjectReference morVolumeDatastore = getTargetDatastoreMOReference(filerTo.getUuid(), dsHost);
                      if (morVolumeDatastore == null) {
                          String msg = String.format("Unable to find the target datastore: %s in datacenter: %s to execute migration", filerTo.getUuid(), dcMo.getName());
                          logger.error(msg);
                          throw new CloudRuntimeException(msg);
                      }
  
                      String mountedDs = getMountedDatastoreName(sourceHyperHost, srcHostApiVersion, filerTo);
                      if (mountedDs != null) {
                          mountedDatastoresAtSource.add(mountedDs);
                      }
  
                      if (volume.getType() == Volume.Type.ROOT) {
                          relocateSpec.setDatastore(morVolumeDatastore);
                      }
                      VirtualMachineRelocateSpecDiskLocator diskLocator = new VirtualMachineRelocateSpecDiskLocator();
                      diskLocator.setDatastore(morVolumeDatastore);
                      Pair<VirtualDisk, String> diskInfo = getVirtualDiskInfo(vmMo, volume.getPath() + VMDK_EXTENSION);
                      String vmdkAbsFile = VmwareHelper.getAbsoluteVmdkFile(diskInfo.first());
                      if (vmdkAbsFile != null && !vmdkAbsFile.isEmpty()) {
                          vmMo.updateAdapterTypeIfRequired(vmdkAbsFile);
                      }
                      int diskId = diskInfo.first().getKey();
                      diskLocator.setDiskId(diskId);
  
                      diskLocators.add(diskLocator);
                      volumeDeviceKey.put(volume.getId(), diskId);
                  }
                  // If a target datastore is provided for the VM, then by default all volumes associated with the VM will be migrated to that target datastore.
                  // Hence set the existing datastore as target datastore for volumes that are not to be migrated.
                  List<Pair<Integer, ManagedObjectReference>> diskDatastores = vmMo.getAllDiskDatastores();
                  for (Pair<Integer, ManagedObjectReference> diskDatastore : diskDatastores) {
                      if (!volumeDeviceKey.containsValue(diskDatastore.first())) {
                          VirtualMachineRelocateSpecDiskLocator diskLocator = new VirtualMachineRelocateSpecDiskLocator();
                          diskLocator.setDiskId(diskDatastore.first());
                          diskLocator.setDatastore(diskDatastore.second());
                          diskLocators.add(diskLocator);
                      }
                  }
  
                  relocateSpec.getDisk().addAll(diskLocators);
              }
  
              // Specific section for MigrateVmWithStorageCommand
              if (vmTo != null) {
                  // Prepare network at target before migration
                  NicTO[] nics = vmTo.getNics();
                  for (NicTO nic : nics) {
                      // prepare network on the host
                      prepareNetworkFromNicInfo((HostMO)targetHyperHost, nic, false,
                              vmTo.getNetworkIdToNetworkNameMap().get(nic.getNetworkId()), vmTo.getType());
                  }
  
                  if (targetHyperHost == null) {
                      throw new CloudRuntimeException(String.format("Trying to relocate VM [%s], but target hyper host is null.", vmTo.getUuid()));
                  }
  
                  // Ensure secondary storage mounted on target host
                  VmwareManager mgr = targetHyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
                  Pair<String, Long> secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId));
                  String secStoreUrl = secStoreUrlAndId.first();
                  if (secStoreUrl == null) {
                      String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used",
                              _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100));
                      throw new Exception(msg);
                  }
                  ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, targetHyperHost);
                  if (morSecDs == null) {
                      throw new Exception(String.format("Failed to prepare secondary storage on host, NFS secondary or cache store url: %s in dc %s", secStoreUrl, _dcId));
                  }
              }
  
              if (srcHostApiVersion.compareTo("5.1") < 0) {
                  // Migrate VM's volumes to target datastore(s).
                  if (!vmMo.changeDatastore(relocateSpec)) {
                      throw new Exception("Change datastore operation failed during storage migration");
                  } else {
                      logger.debug(String.format("Successfully migrated storage of VM: %s to target datastore(s)", vmName));
                  }
                  // Migrate VM to target host.
                  if (targetHyperHost != null) {
                      ManagedObjectReference morPool = targetHyperHost.getHyperHostOwnerResourcePool();
                      if (!vmMo.migrate(morPool, targetHyperHost.getMor())) {
                          throw new Exception("VM migration to target host failed during storage migration");
                      } else {
                          logger.debug(String.format("Successfully migrated VM: %s from host %s to %s", vmName , sourceHyperHost.getHyperHostName(), targetHyperHost.getHyperHostName()));
                      }
                  }
              } else {
                  // Add target host to relocate spec
                  if (targetHyperHost != null) {
                      relocateSpec.setHost(targetHyperHost.getMor());
                      relocateSpec.setPool(targetHyperHost.getHyperHostOwnerResourcePool());
                  }
                  if (!vmMo.changeDatastore(relocateSpec)) {
                      throw new Exception("Change datastore operation failed during storage migration");
                  } else {
                      String msg = String.format("Successfully migrated VM: %s with its storage to target datastore(s)", vmName);
                      if (targetHyperHost != null) {
                          msg = String.format("%s from host %s to %s", msg, sourceHyperHost.getHyperHostName(), targetHyperHost.getHyperHostName());
                      }
                      logger.debug(msg);
                  }
              }
  
              // Consolidate VM disks.
              // In case of a linked clone VM, if VM's disks are not consolidated, further VM operations such as volume snapshot, VM snapshot etc. will result in DB inconsistencies.
              if (!vmMo.consolidateVmDisks()) {
                  logger.warn("VM disk consolidation failed after storage migration. Yet proceeding with VM migration.");
              } else {
                  logger.debug(String.format("Successfully consolidated disks of VM: %s", vmName));
              }
  
              if (MapUtils.isNotEmpty(volumeDeviceKey)) {
                  // Update and return volume path and chain info for every disk because that could have changed after migration
                  VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
                  for (Pair<VolumeTO, StorageFilerTO> entry : volToFiler) {
                      final VolumeTO volume = entry.first();
                      final long volumeId = volume.getId();
                      VirtualDisk[] disks = vmMo.getAllDiskDevice();
                      for (VirtualDisk disk : disks) {
                          if (volumeDeviceKey.get(volumeId) == disk.getKey()) {
                              VolumeObjectTO newVol = new VolumeObjectTO();
                              newVol.setDataStoreUuid(entry.second().getUuid());
                              String newPath = vmMo.getVmdkFileBaseName(disk);
                              ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(targetHyperHost != null ? targetHyperHost : sourceHyperHost, entry.second().getUuid());
                              DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDs);
                              VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(newPath, dsMo.getName());
                              newVol.setId(volumeId);
                              newVol.setPath(newPath);
                              newVol.setChainInfo(_gson.toJson(diskInfo));
                              volumeToList.add(newVol);
                              break;
                          }
                      }
                  }
              }
          } catch (Throwable e) {
              if (e instanceof RemoteException) {
                  logger.warn("Encountered remote exception at vCenter, invalidating VMware session context");
                  invalidateServiceContext();
              }
              throw e;
          } finally {
              // Cleanup datastores mounted on source host
              for (String mountedDatastore : mountedDatastoresAtSource) {
                  logger.debug("Attempting to unmount datastore " + mountedDatastore + " at " + sourceHyperHost.getHyperHostName());
                  try {
                      sourceHyperHost.unmountDatastore(mountedDatastore);
                  } catch (Exception unmountEx) {
                      logger.warn("Failed to unmount datastore " + mountedDatastore + " at " + sourceHyperHost.getHyperHostName() + ". Seems the datastore is still being used by " + sourceHyperHost.getHyperHostName() +
                              ". Please unmount manually to cleanup.");
                  }
                  logger.debug("Successfully unmounted datastore " + mountedDatastore + " at " + sourceHyperHost.getHyperHostName());
              }
          }
  
          // Only when volToFiler is not empty a filled list of VolumeObjectTO is returned else it will be empty
          return volumeToList;
      }
  
      private String getMountedDatastoreName(VmwareHypervisorHost sourceHyperHost, String sourceHostApiVersion, StorageFilerTO filerTo) throws Exception {
          String mountedDatastoreName = null;
          // If host version is below 5.1 then simultaneous change of VM's datastore and host is not supported.
          // So since only the datastore will be changed first, ensure the target datastore is mounted on source host.
          if (sourceHostApiVersion.compareTo("5.1") < 0) {
              logger.debug(String.format("Host: %s version is %s, vMotion without shared storage cannot be done. Check source host has target datastore mounted or can be mounted", sourceHyperHost.getHyperHostName(), sourceHostApiVersion));
              ManagedObjectReference morVolumeDatastoreAtSource = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(sourceHyperHost, filerTo.getUuid());
              String volumeDatastoreName = filerTo.getUuid().replace("-", "");
              String volumeDatastoreHost = filerTo.getHost();
              String volumeDatastorePath = filerTo.getPath();
              int volumeDatastorePort = filerTo.getPort();
  
              // If datastore is NFS and target datastore is not already mounted on source host then mount the datastore.
              if (filerTo.getType().equals(StoragePoolType.NetworkFilesystem)) {
                  if (morVolumeDatastoreAtSource == null) {
                      morVolumeDatastoreAtSource = sourceHyperHost.mountDatastore(false, volumeDatastoreHost, volumeDatastorePort, volumeDatastorePath, volumeDatastoreName, false);
                      if (morVolumeDatastoreAtSource == null) {
                          throw new Exception("Unable to mount NFS datastore " + volumeDatastoreHost + ":/" + volumeDatastorePath + " on host: " + sourceHyperHost.getHyperHostName());
                      }
                      mountedDatastoreName = volumeDatastoreName;
                      logger.debug("Mounted NFS datastore " + volumeDatastoreHost + ":/" + volumeDatastorePath + " on host: " + sourceHyperHost.getHyperHostName());
                  }
              }
  
              // If datastore is VMFS and target datastore is not mounted or accessible to source host then fail migration.
              if (filerTo.getType().equals(StoragePoolType.VMFS)) {
                  if (morVolumeDatastoreAtSource == null) {
                      logger.warn("Host: " + sourceHyperHost.getHyperHostName() + " version is below 5.1, target VMFS datastore(s) need to be manually mounted on host for successful storage migration.");
                      throw new Exception("Target VMFS datastore: " + volumeDatastorePath + " is not mounted on host: " + sourceHyperHost.getHyperHostName());
                  }
                  DatastoreMO dsAtSourceMo = new DatastoreMO(getServiceContext(), morVolumeDatastoreAtSource);
                  String srcHostValue = sourceHyperHost.getMor().getValue();
                  if (!dsAtSourceMo.isAccessibleToHost(srcHostValue)) {
                      logger.warn("Host " + sourceHyperHost.getHyperHostName() + " version is below 5.1, target VMFS datastore(s) need to be accessible to host for a successful storage migration.");
                      throw new Exception("Target VMFS datastore: " + volumeDatastorePath + " is not accessible on host: " + sourceHyperHost.getHyperHostName());
                  }
              }
          }
          return mountedDatastoreName;
      }
  
      private Answer execute(ValidateVcenterDetailsCommand cmd) {
          String vCenterServerAddress = cmd.getvCenterServerAddress();
          VmwareContext context = getServiceContext();
  
          if (vCenterServerAddress.equals(context.getServerAddress())) {
              return new Answer(cmd, true, "success");
          } else {
              return new Answer(cmd, false, "Provided vCenter server address is invalid");
          }
      }
  
      public String acquireVirtualMachineVncTicket(String vmInternalCSName) throws Exception {
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter());
          VirtualMachineMO vmMo = dcMo.findVm(vmInternalCSName);
          return vmMo.acquireVncTicket();
      }
  
      private GetVmVncTicketAnswer execute(GetVmVncTicketCommand cmd) {
          String vmInternalName = cmd.getVmInternalName();
          logger.info("Getting VNC ticket for VM " + vmInternalName);
          try {
              String ticket = acquireVirtualMachineVncTicket(vmInternalName);
              boolean result = StringUtils.isNotBlank(ticket);
              return new GetVmVncTicketAnswer(ticket, result, result ? "" : "Empty ticket obtained");
          } catch (Exception e) {
              logger.error("Error getting VNC ticket for VM " + vmInternalName, e);
              return new GetVmVncTicketAnswer(null, false, e.getLocalizedMessage());
          }
      }
  
      protected CheckGuestOsMappingAnswer execute(CheckGuestOsMappingCommand cmd) {
          String guestOsName = cmd.getGuestOsName();
          String guestOsMappingName = cmd.getGuestOsHypervisorMappingName();
          logger.info("Checking guest os mapping name: " + guestOsMappingName + " for the guest os: " + guestOsName + " in the hypervisor");
          try {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              GuestOsDescriptor guestOsDescriptor = hyperHost.getGuestOsDescriptor(guestOsMappingName);
              if (guestOsDescriptor == null) {
                  return new CheckGuestOsMappingAnswer(cmd, "Guest os mapping name: " + guestOsMappingName + " not found in the hypervisor");
              }
              logger.debug("Matching hypervisor guest os - id: " + guestOsDescriptor.getId() + ", full name: " + guestOsDescriptor.getFullName() + ", family: " + guestOsDescriptor.getFamily());
              if (guestOsDescriptor.getFullName().equalsIgnoreCase(guestOsName)) {
                  logger.debug("Hypervisor guest os name in the descriptor matches with os name: " + guestOsName);
              }
              logger.info("Hypervisor guest os name in the descriptor matches with os mapping: " + guestOsMappingName + " from user");
              return new CheckGuestOsMappingAnswer(cmd);
          } catch (Exception e) {
              logger.error("Failed to check the hypervisor guest os mapping name: " + guestOsMappingName, e);
              return new CheckGuestOsMappingAnswer(cmd, e.getLocalizedMessage());
          }
      }
  
      protected ListDataStoreObjectsAnswer execute(ListDataStoreObjectsCommand cmd) {
          String path = cmd.getPath();
          int startIndex = cmd.getStartIndex();
          int pageSize = cmd.getPageSize();
          PrimaryDataStoreTO dataStore = (PrimaryDataStoreTO) cmd.getStore();
  
          if (path.startsWith("/")) {
              path = path.substring(1);
          }
  
          if (path.endsWith("/")) {
              path = path.substring(0, path.length() - 1);
          }
  
          VmwareContext context = getServiceContext();
          VmwareHypervisorHost hyperHost = getHyperHost(context);
          ManagedObjectReference morDatastore;
  
          int count = 0;
          List<String> names = new ArrayList<>();
          List<String> paths = new ArrayList<>();
          List<String> absPaths = new ArrayList<>();
          List<Boolean> isDirs = new ArrayList<>();
          List<Long> sizes = new ArrayList<>();
          List<Long> modifiedList = new ArrayList<>();
  
          try {
              morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, dataStore.getUuid());
  
              DatastoreMO dsMo = new DatastoreMO(context, morDatastore);
              HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO();
              FileQueryFlags fqf = new FileQueryFlags();
              fqf.setFileSize(true);
              fqf.setFileType(true);
              fqf.setModification(true);
              fqf.setFileOwner(false);
  
              HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec();
              spec.setSearchCaseInsensitive(true);
              spec.setDetails(fqf);
  
              String dsPath = String.format("[%s] %s", dsMo.getName(), path);
  
              HostDatastoreBrowserSearchResults results = browserMo.searchDatastore(dsPath, spec);
              List<FileInfo> fileInfoList = results.getFile();
              count = fileInfoList.size();
              for (int i = startIndex; i < startIndex + pageSize && i < count; i++) {
                  FileInfo file = fileInfoList.get(i);
  
                  names.add(file.getPath());
                  paths.add(path + "/" + file.getPath());
                  absPaths.add(dsPath + "/" + file.getPath());
                  isDirs.add(file instanceof FolderFileInfo);
                  sizes.add(file.getFileSize());
                  modifiedList.add(file.getModification().toGregorianCalendar().getTimeInMillis());
              }
  
              return new ListDataStoreObjectsAnswer(true, count, names, paths, absPaths, isDirs, sizes, modifiedList);
          } catch (Exception e) {
              if (e.getMessage().contains("was not found")) {
                  return new ListDataStoreObjectsAnswer(false, count, names, paths, absPaths, isDirs, sizes, modifiedList);
              }
              String errorMsg = String.format("Failed to list files at path [%s] due to: [%s].", path, e.getMessage());
              logger.error(errorMsg, e);
          }
  
          return null;
      }
  
      protected GetHypervisorGuestOsNamesAnswer execute(GetHypervisorGuestOsNamesCommand cmd) {
          String keyword = cmd.getKeyword();
          logger.info("Getting guest os names in the hypervisor");
          try {
              VmwareContext context = getServiceContext();
              VmwareHypervisorHost hyperHost = getHyperHost(context);
              List<GuestOsDescriptor> guestOsDescriptors = hyperHost.getGuestOsDescriptors();
              if (guestOsDescriptors == null) {
                  return new GetHypervisorGuestOsNamesAnswer(cmd, "Guest os names not found in the hypervisor");
              }
              List<Pair<String, String>> hypervisorGuestOsNames = new ArrayList<>();
              for (GuestOsDescriptor guestOsDescriptor : guestOsDescriptors) {
                  String osDescriptorFullName = guestOsDescriptor.getFullName();
                  String osDescriptorId = guestOsDescriptor.getId();
                  if (StringUtils.isNotBlank(keyword)) {
                      if (osDescriptorFullName.toLowerCase().contains(keyword.toLowerCase()) || osDescriptorId.toLowerCase().contains(keyword.toLowerCase())) {
                          Pair<String, String> hypervisorGuestOs = new Pair<>(osDescriptorFullName, osDescriptorId);
                          hypervisorGuestOsNames.add(hypervisorGuestOs);
                      }
                  } else {
                      Pair<String, String> hypervisorGuestOs = new Pair<>(osDescriptorFullName, osDescriptorId);
                      hypervisorGuestOsNames.add(hypervisorGuestOs);
                  }
              }
              return new GetHypervisorGuestOsNamesAnswer(cmd, hypervisorGuestOsNames);
          } catch (Exception e) {
              logger.error("Failed to get the hypervisor guest names due to: " + e.getLocalizedMessage(), e);
              return new GetHypervisorGuestOsNamesAnswer(cmd, e.getLocalizedMessage());
          }
      }
  
      private Answer execute(PrepareForBackupRestorationCommand command) {
          try {
              VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
  
              String vmName = command.getVmName();
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  if (hyperHost instanceof HostMO) {
                      ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
                      vmMo = clusterMo.findVmOnHyperHost(vmName);
                  }
              }
  
              if (vmMo == null) {
                  String msg = "VM " + vmName + " no longer exists to execute PrepareForBackupRestorationCommand command";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              vmMo.removeChangeTrackPathFromVmdkForDisks();
  
              return new Answer(command, true, "success");
          } catch (Exception e) {
              logger.error("Unexpected exception: ", e);
              return new Answer(command, false, "Unable to execute PrepareForBackupRestorationCommand due to " + e);
          }
      }
  
      private Integer getVmwareWindowTimeInterval() {
          Integer windowInterval = VmwareManager.VMWARE_STATS_TIME_WINDOW.value();
          if (windowInterval == null || windowInterval < 20) {
              logger.error(String.format("The window interval can't be [%s]. Therefore we will use the default value of [%s] seconds.", windowInterval, VmwareManager.VMWARE_STATS_TIME_WINDOW.defaultValue()));
              windowInterval = Integer.valueOf(VmwareManager.VMWARE_STATS_TIME_WINDOW.defaultValue());
          }
          return windowInterval;
      }
  
      @Override
      public String createLogMessageException(Throwable e, Command command) {
          if (e instanceof RemoteException) {
              logger.warn("Encounter remote exception to vCenter, invalidate VMware session context.");
              invalidateServiceContext();
          }
  
          String message = String.format("%s failed due to [%s].", command.getClass().getSimpleName(), VmwareHelper.getExceptionMessage(e));
          logger.error(message, e);
  
          return message;
      }
  
      private void logCommand(Command cmd) {
          try {
              logger.debug(String.format(EXECUTING_RESOURCE_COMMAND, cmd.getClass().getSimpleName(), _gson.toJson(cmd)));
          } catch (Exception e) {
              logger.error(String.format("Failed to log command %s due to: [%s].", cmd.getClass().getSimpleName(), e.getMessage()), e);
          }
      }
  
      @Override
      public VmwareStorageProcessor getStorageProcessor() {
          return _storageProcessor;
      }
  }
  