content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.hypervisor.vmware.manager;
  
  import java.io.BufferedWriter;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileOutputStream;
  import java.io.OutputStreamWriter;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.Properties;
  import java.util.UUID;
  
  import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
  import org.apache.cloudstack.storage.to.TemplateObjectTO;
  import org.apache.cloudstack.storage.to.VolumeObjectTO;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.logging.log4j.Logger;
  import org.apache.logging.log4j.LogManager;
  
  import com.cloud.agent.api.Answer;
  import com.cloud.agent.api.BackupSnapshotAnswer;
  import com.cloud.agent.api.BackupSnapshotCommand;
  import com.cloud.agent.api.CreatePrivateTemplateFromSnapshotCommand;
  import com.cloud.agent.api.CreatePrivateTemplateFromVolumeCommand;
  import com.cloud.agent.api.CreateVMSnapshotAnswer;
  import com.cloud.agent.api.CreateVMSnapshotCommand;
  import com.cloud.agent.api.CreateVolumeFromSnapshotAnswer;
  import com.cloud.agent.api.CreateVolumeFromSnapshotCommand;
  import com.cloud.agent.api.DeleteVMSnapshotAnswer;
  import com.cloud.agent.api.DeleteVMSnapshotCommand;
  import com.cloud.agent.api.RevertToVMSnapshotAnswer;
  import com.cloud.agent.api.RevertToVMSnapshotCommand;
  import com.cloud.agent.api.storage.CopyVolumeAnswer;
  import com.cloud.agent.api.storage.CopyVolumeCommand;
  import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
  import com.cloud.agent.api.storage.CreatePrivateTemplateAnswer;
  import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer;
  import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
  import com.cloud.agent.api.to.DataObjectType;
  import com.cloud.agent.api.to.DataStoreTO;
  import com.cloud.agent.api.to.DataTO;
  import com.cloud.agent.api.to.DiskTO;
  import com.cloud.agent.api.to.NfsTO;
  import com.cloud.agent.api.to.StorageFilerTO;
  import com.cloud.hypervisor.vmware.mo.CustomFieldConstants;
  import com.cloud.hypervisor.vmware.mo.DatacenterMO;
  import com.cloud.hypervisor.vmware.mo.DatastoreFile;
  import com.cloud.hypervisor.vmware.mo.DatastoreMO;
  import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO;
  import com.cloud.hypervisor.vmware.mo.HostMO;
  import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
  import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
  import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost;
  import com.cloud.hypervisor.vmware.util.VmwareContext;
  import com.cloud.hypervisor.vmware.util.VmwareHelper;
  import com.cloud.storage.JavaStorageLayer;
  import com.cloud.storage.Storage;
  import com.cloud.storage.Storage.ImageFormat;
  import com.cloud.storage.StorageLayer;
  import com.cloud.storage.Volume;
  import com.cloud.storage.resource.VmwareStorageProcessor;
  import com.cloud.storage.template.OVAProcessor;
  import com.cloud.utils.NumbersUtil;
  import com.cloud.utils.Pair;
  import com.cloud.utils.Ternary;
  import com.cloud.utils.UuidUtils;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.script.Script;
  import com.cloud.vm.VirtualMachine;
  import com.cloud.vm.snapshot.VMSnapshot;
  import com.vmware.vim25.FileInfo;
  import com.vmware.vim25.FileQueryFlags;
  import com.vmware.vim25.HostDatastoreBrowserSearchResults;
  import com.vmware.vim25.HostDatastoreBrowserSearchSpec;
  import com.vmware.vim25.ManagedObjectReference;
  import com.vmware.vim25.TaskInfo;
  import com.vmware.vim25.TaskInfoState;
  import com.vmware.vim25.VirtualDisk;
  
  public class VmwareStorageManagerImpl implements VmwareStorageManager {
  
      private String _nfsVersion;
  
  
      @Override
      public boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd) {
          DataTO data = cmd.getData();
          int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()),
                      Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier());
          if (data == null) {
              return false;
          }
  
          String newPath = null;
          if (data.getObjectType() == DataObjectType.VOLUME) {
              newPath = createOvaForVolume((VolumeObjectTO)data, timeout);
          } else if (data.getObjectType() == DataObjectType.TEMPLATE) {
              newPath = createOvaForTemplate((TemplateObjectTO)data, timeout);
          } else if (data.getObjectType() == DataObjectType.ARCHIVE) {
              newPath = cmd.getInstallPath();
          }
          if (newPath != null) {
              cmd.setInstallPath(newPath);
              return true;
          }
          return false;
  
      }
  
      @Override
      public void createOva(String path, String name, int archiveTimeout) {
          Script commandSync = new Script(true, "sync", 0, logger);
          commandSync.execute();
  
          Script command = new Script(false, "tar", archiveTimeout, logger);
          command.setWorkDir(path);
          command.add("-cf", name + ".ova");
          command.add(name + ".ovf");        // OVF file should be the first file in OVA archive
          command.add(name + "-disk0.vmdk");
  
          logger.info("Package OVA with command: " + command.toString());
          command.execute();
      }
  
      protected Logger logger = LogManager.getLogger(getClass());
  
      private final VmwareStorageMount _mountService;
      private final StorageLayer _storage = new JavaStorageLayer();
  
      private int _timeout;
  
      public VmwareStorageManagerImpl(VmwareStorageMount mountService) {
          assert (mountService != null);
          _mountService = mountService;
      }
  
      public VmwareStorageManagerImpl(VmwareStorageMount mountService, String nfsVersion) {
          assert (mountService != null);
          _mountService = mountService;
          _nfsVersion = nfsVersion;
      }
  
      public void configure(Map<String, Object> params) {
          logger.info("Configure VmwareStorageManagerImpl");
  
          String value = (String)params.get("scripts.timeout");
          _timeout = NumbersUtil.parseInt(value, 1440) * 1000;
      }
  
      @Override
      public String createOvaForTemplate(TemplateObjectTO template, int archiveTimeout) {
          DataStoreTO storeTO = template.getDataStore();
          if (!(storeTO instanceof NfsTO)) {
              logger.debug("Can only handle NFS storage, while creating OVA from template");
              return null;
          }
          NfsTO nfsStore = (NfsTO)storeTO;
          String secStorageUrl = nfsStore.getUrl();
          assert (secStorageUrl != null);
          String installPath = template.getPath();
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsStore.getNfsVersion());
          String installFullPath = secondaryMountPoint + "/" + installPath;
          try {
              if (installFullPath.endsWith(".ova")) {
                  if (new File(installFullPath).exists()) {
                      logger.debug("OVA file found at: " + installFullPath);
                  } else {
                      if (new File(installFullPath + ".meta").exists()) {
                          createOVAFromMetafile(installFullPath + ".meta", archiveTimeout);
                      } else {
                          String msg = "Unable to find OVA or OVA MetaFile to prepare template.";
                          logger.error(msg);
                          throw new Exception(msg);
                      }
                  }
                  return installPath;
              }
          } catch (Throwable e) {
              logger.debug("Failed to create OVA: " + e.toString());
          }
          return null;
      }
  
      //Fang: new command added;
      // Important! we need to sync file system before we can safely use tar to work around a linux kernel bug(or feature)
      public String createOvaForVolume(VolumeObjectTO volume, int archiveTimeout) {
          DataStoreTO storeTO = volume.getDataStore();
          if (!(storeTO instanceof NfsTO)) {
              logger.debug("can only handle nfs storage, when create ova from volume");
              return null;
          }
          NfsTO nfsStore = (NfsTO)storeTO;
          String secStorageUrl = nfsStore.getUrl();
          assert (secStorageUrl != null);
          //Note the volume path is volumes/accountId/volumeId/uuid/, the actual volume is uuid/uuid.vmdk
          String installPath = volume.getPath();
          int index = installPath.lastIndexOf(File.separator);
          String volumeUuid = installPath.substring(index + 1);
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsStore.getNfsVersion());
          //The real volume path
          String volumePath = installPath + File.separator + volumeUuid + ".ova";
          String installFullPath = secondaryMountPoint + "/" + installPath;
  
          try {
              if (new File(secondaryMountPoint + File.separator + volumePath).exists()) {
                  logger.debug("ova already exists:" + volumePath);
                  return volumePath;
              } else {
                  Script commandSync = new Script(true, "sync", 0, logger);
                  commandSync.execute();
                  Script command = new Script(false, "tar", archiveTimeout, logger);
                  command.setWorkDir(installFullPath);
                  command.add("-cf", volumeUuid + ".ova");
                  command.add(volumeUuid + ".ovf");        // OVF file should be the first file in OVA archive
                  command.add(volumeUuid + "-disk0.vmdk");
  
                  String result = command.execute();
                  if (result != Script.ERR_TIMEOUT) {
                      return volumePath;
                  }
  
              }
          } catch (Throwable e) {
              logger.info("Exception for createVolumeOVA");
          }
          return null;
      }
  
      @Override
      public Answer execute(VmwareHostService hostService, PrimaryStorageDownloadCommand cmd) {
          String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
          assert (secondaryStorageUrl != null);
  
          String templateUrl = cmd.getUrl();
  
          String templateName = null;
          String mountPoint = null;
          if (templateUrl.endsWith(".ova")) {
              int index = templateUrl.lastIndexOf("/");
              mountPoint = templateUrl.substring(0, index);
              mountPoint = mountPoint.substring(secondaryStorageUrl.length() + 1);
              if (!mountPoint.endsWith("/")) {
                  mountPoint = mountPoint + "/";
              }
  
              templateName = templateUrl.substring(index + 1).replace("." + ImageFormat.OVA.getFileExtension(), "");
  
              if (templateName == null || templateName.isEmpty()) {
                  templateName = cmd.getName();
              }
          } else {
              mountPoint = templateUrl.substring(secondaryStorageUrl.length() + 1);
              if (!mountPoint.endsWith("/")) {
                  mountPoint = mountPoint + "/";
              }
              templateName = cmd.getName();
          }
  
          VmwareContext context = hostService.getServiceContext(cmd);
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
  
              String templateUuidName = UuidUtils.nameUUIDFromBytes((templateName + "@" + cmd.getPoolUuid() + "-" + hyperHost.getMor().getValue()).getBytes("UTF-8")).toString();
              // truncate template name to 32 chars to ensure they work well with vSphere API's.
              templateUuidName = templateUuidName.replace("-", "");
  
              DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter());
              VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true);
  
              if (templateMo == null) {
                  if (logger.isInfoEnabled()) {
                      logger.info("Template " + templateName + " is not setup yet, setup template from secondary storage with uuid name: " + templateUuidName);
                  }
                  ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPoolUuid());
                  assert (morDs != null);
                  DatastoreMO primaryStorageDatastoreMo = new DatastoreMO(context, morDs);
  
                  copyTemplateFromSecondaryToPrimary(hyperHost, primaryStorageDatastoreMo, secondaryStorageUrl, mountPoint, templateName, templateUuidName, cmd.getNfsVersion());
              } else {
                  logger.info("Template " + templateName + " has already been setup, skip the template setup process in primary storage");
              }
  
              return new PrimaryStorageDownloadAnswer(templateUuidName, 0);
          } catch (Throwable e) {
              return new PrimaryStorageDownloadAnswer(hostService.createLogMessageException(e, cmd));
          }
      }
  
      @Override
      @Deprecated
      public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd) {
          Long accountId = cmd.getAccountId();
          Long volumeId = cmd.getVolumeId();
          String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
          String snapshotUuid = cmd.getSnapshotUuid(); // not null: Precondition.
          String prevSnapshotUuid = cmd.getPrevSnapshotUuid();
          String prevBackupUuid = cmd.getPrevBackupUuid();
          String searchExcludedFolders = cmd.getContextParam("searchexludefolders");
          VirtualMachineMO workerVm = null;
          String workerVMName = null;
          String volumePath = cmd.getVolumePath();
          ManagedObjectReference morDs = null;
          DatastoreMO dsMo = null;
  
          // By default assume failure
          String details = null;
          boolean success = false;
          String snapshotBackupUuid = null;
  
          VmwareContext context = hostService.getServiceContext(cmd);
          VirtualMachineMO vmMo = null;
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
              morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid());
  
              try {
                  vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
                  if (vmMo == null) {
                      if (logger.isDebugEnabled()) {
                          logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter");
                      }
  
                      vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName());
                      if (vmMo == null) {
                          dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
  
                          workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo);
                          vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null);
  
                          if (vmMo == null) {
                              throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName);
                          }
                          workerVm = vmMo;
  
                          // attach volume to worker VM
                          String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk", searchExcludedFolders);
                          vmMo.attachDisk(new String[] {datastoreVolumePath}, morDs);
                      }
                  }
  
                  if (!vmMo.createSnapshot(snapshotUuid, "Snapshot taken for " + cmd.getSnapshotName(), false, false)) {
                      throw new Exception("Failed to take snapshot " + cmd.getSnapshotName() + " on vm: " + cmd.getVmName());
                  }
  
                  snapshotBackupUuid = backupSnapshotToSecondaryStorage(vmMo, accountId, volumeId, cmd.getVolumePath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid,
                          prevBackupUuid, hostService.getWorkerName(context, cmd, 1, dsMo), cmd.getNfsVersion());
  
                  success = (snapshotBackupUuid != null);
                  if (success) {
                      details = "Successfully backedUp the snapshotUuid: " + snapshotUuid + " to secondary storage.";
                  }
  
              } finally {
                  if (vmMo != null) {
                      ManagedObjectReference snapshotMor = vmMo.getSnapshotMor(snapshotUuid);
                      if (snapshotMor != null) {
                          vmMo.removeSnapshot(snapshotUuid, false);
                      }
                  }
  
                  try {
                      if (workerVm != null) {
                          workerVm.detachAllDisksAndDestroy();
                      }
                  } catch (Throwable e) {
                      logger.warn(String.format("Failed to destroy worker VM [%s] due to: [%s].", workerVMName, e.getMessage()), e);
                  }
              }
          } catch (Throwable e) {
              return new BackupSnapshotAnswer(cmd, false, hostService.createLogMessageException(e, cmd), snapshotBackupUuid, true);
          }
  
          return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, true);
      }
  
      @Override
      public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromVolumeCommand cmd) {
          String secondaryStoragePoolURL = cmd.getSecondaryStorageUrl();
          String volumePath = cmd.getVolumePath();
          Long accountId = cmd.getAccountId();
          Long templateId = cmd.getTemplateId();
          String details = null;
  
          VmwareContext context = hostService.getServiceContext(cmd);
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
  
              VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
              if (vmMo == null) {
                  if (logger.isDebugEnabled()) {
                      logger.debug("Unable to find the owner VM for CreatePrivateTemplateFromVolumeCommand on host " + hyperHost.getHyperHostName() + ", try within datacenter");
                  }
                  vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName());
  
                  if (vmMo == null) {
                      String msg = "Unable to find the owner VM for volume operation. vm: " + cmd.getVmName();
                      logger.error(msg);
                      throw new Exception(msg);
                  }
              }
  
              Ternary<String, Long, Long> result = createTemplateFromVolume(vmMo, accountId, templateId, cmd.getUniqueName(), secondaryStoragePoolURL, volumePath,
                      hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion());
  
              return new CreatePrivateTemplateAnswer(cmd, true, null, result.first(), result.third(), result.second(), cmd.getUniqueName(), ImageFormat.OVA);
  
          } catch (Throwable e) {
              return new CreatePrivateTemplateAnswer(cmd, false, hostService.createLogMessageException(e, cmd));
          }
      }
  
      @Override
      public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromSnapshotCommand cmd) {
          Long accountId = cmd.getAccountId();
          Long volumeId = cmd.getVolumeId();
          String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
          String backedUpSnapshotUuid = cmd.getSnapshotUuid();
          Long newTemplateId = cmd.getNewTemplateId();
          String details;
          String uniqeName = UUID.randomUUID().toString();
  
          VmwareContext context = hostService.getServiceContext(cmd);
          try {
              Ternary<String, Long, Long> result = createTemplateFromSnapshot(accountId, newTemplateId, uniqeName, secondaryStorageUrl, volumeId, backedUpSnapshotUuid,
                      cmd.getNfsVersion());
  
              return new CreatePrivateTemplateAnswer(cmd, true, null, result.first(), result.third(), result.second(), uniqeName, ImageFormat.OVA);
          } catch (Throwable e) {
              return new CreatePrivateTemplateAnswer(cmd, false, hostService.createLogMessageException(e, cmd));
          }
      }
  
      @Override
      public Answer execute(VmwareHostService hostService, CopyVolumeCommand cmd) {
          Long volumeId = cmd.getVolumeId();
          String volumePath = cmd.getVolumePath();
          String secondaryStorageURL = cmd.getSecondaryStorageURL();
          String vmName = cmd.getVmName();
  
          VmwareContext context = hostService.getServiceContext(cmd);
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
  
              Pair<String, String> result;
              if (cmd.toSecondaryStorage()) {
                  result = copyVolumeToSecStorage(hostService, hyperHost, cmd, vmName, volumeId, cmd.getPool().getUuid(), volumePath, secondaryStorageURL,
                          hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion());
              } else {
                  StorageFilerTO poolTO = cmd.getPool();
  
                  ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolTO.getUuid());
                  if (morDatastore == null) {
                      morDatastore = hyperHost.mountDatastore(false, poolTO.getHost(), 0, poolTO.getPath(), poolTO.getUuid().replace("-", ""), true);
  
                      if (morDatastore == null) {
                          throw new Exception("Unable to mount storage pool on host. storeUrl: " + poolTO.getHost() + ":/" + poolTO.getPath());
                      }
                  }
  
                  result = copyVolumeFromSecStorage(hyperHost, volumeId, new DatastoreMO(context, morDatastore), secondaryStorageURL, volumePath, cmd.getNfsVersion());
                  deleteVolumeDirOnSecondaryStorage(volumeId, secondaryStorageURL, cmd.getNfsVersion());
              }
              return new CopyVolumeAnswer(cmd, true, null, result.first(), result.second());
          } catch (Throwable e) {
              return new CopyVolumeAnswer(cmd, false, hostService.createLogMessageException(e, cmd), null, null);
          }
      }
  
      @Override
      public Answer execute(VmwareHostService hostService, CreateVolumeFromSnapshotCommand cmd) {
  
          String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel();
          Long accountId = cmd.getAccountId();
          Long volumeId = cmd.getVolumeId();
          String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
          String backedUpSnapshotUuid = cmd.getSnapshotUuid();
  
          String details = null;
          boolean success = false;
          String newVolumeName = UUID.randomUUID().toString().replace("-", "");
  
          VmwareContext context = hostService.getServiceContext(cmd);
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
              ManagedObjectReference morPrimaryDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, primaryStorageNameLabel);
              if (morPrimaryDs == null) {
                  String msg = "Unable to find datastore: " + primaryStorageNameLabel;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs);
              details = createVolumeFromSnapshot(hyperHost, primaryDsMo, newVolumeName, accountId, volumeId, secondaryStorageUrl, backedUpSnapshotUuid, cmd.getNfsVersion());
              if (details == null) {
                  success = true;
              }
          } catch (Throwable e) {
              details = hostService.createLogMessageException(e, cmd);
          }
  
          return new CreateVolumeFromSnapshotAnswer(cmd, success, details, newVolumeName);
      }
  
  
      // templateName: name in secondary storage
      // templateUuid: will be used at hypervisor layer
      private void copyTemplateFromSecondaryToPrimary(VmwareHypervisorHost hyperHost, DatastoreMO datastoreMo, String secondaryStorageUrl, String templatePathAtSecondaryStorage,
                                                      String templateName, String templateUuid, String nfsVersion) throws Exception {
  
          logger.info("Executing copyTemplateFromSecondaryToPrimary. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: "
                  + templatePathAtSecondaryStorage + ", templateName: " + templateName);
  
          String secondaryMountPoint = _mountService.getMountPoint(secondaryStorageUrl, nfsVersion);
          logger.info("Secondary storage mount point: " + secondaryMountPoint);
  
          String srcOVAFileName = secondaryMountPoint + "/" + templatePathAtSecondaryStorage + templateName + "." + ImageFormat.OVA.getFileExtension();
  
          String srcFileName = getOVFFilePath(srcOVAFileName);
          if (srcFileName == null) {
              Script command = new Script("tar", 0, logger);
              command.add("--no-same-owner");
              command.add("-xf", srcOVAFileName);
              command.setWorkDir(secondaryMountPoint + "/" + templatePathAtSecondaryStorage);
              logger.info("Executing command: " + command.toString());
              String result = command.execute();
              if (result != null) {
                  String msg = "Unable to unpack snapshot OVA file at: " + srcOVAFileName;
                  logger.error(msg);
                  throw new Exception(msg);
              }
          }
  
          srcFileName = getOVFFilePath(srcOVAFileName);
          if (srcFileName == null) {
              String msg = "Unable to locate OVF file in template package directory: " + srcOVAFileName;
              logger.error(msg);
              throw new Exception(msg);
          }
  
          String vmName = templateUuid;
          hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin", null);
  
          VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
          if (vmMo == null) {
              String msg = "Failed to import OVA template. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage
                      + ", templateName: " + templateName + ", templateUuid: " + templateUuid;
              logger.error(msg);
              throw new Exception(msg);
          }
  
          if (vmMo.createSnapshot("cloud.template.base", "Base snapshot", false, false)) {
              vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, templateUuid);
              vmMo.markAsTemplate();
          } else {
              vmMo.destroy();
              String msg = "Unable to create base snapshot for template, templateName: " + templateName + ", templateUuid: " + templateUuid;
              logger.error(msg);
              throw new Exception(msg);
          }
      }
  
      private Ternary<String, Long, Long> createTemplateFromVolume(VirtualMachineMO vmMo, long accountId, long templateId, String templateUniqueName, String secStorageUrl,
                                                                   String volumePath, String workerVmName, String nfsVersion) throws Exception {
  
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsVersion);
          String installPath = getTemplateRelativeDirInSecStorage(accountId, templateId);
          String installFullPath = secondaryMountPoint + "/" + installPath;
          synchronized (installPath.intern()) {
              Script command = new Script(false, "mkdir", _timeout, logger);
              command.add("-p");
              command.add(installFullPath);
  
              String result = command.execute();
              if (result != null) {
                  String msg = "unable to prepare template directory: " + installPath + ", storage: " + secStorageUrl + ", error msg: " + result;
                  logger.error(msg);
                  throw new Exception(msg);
              }
          }
  
          VirtualMachineMO clonedVm = null;
          try {
              Pair<VirtualDisk, String> volumeDeviceInfo = vmMo.getDiskDevice(volumePath);
              if (volumeDeviceInfo == null) {
                  String msg = "Unable to find related disk device for volume. volume path: " + volumePath;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              if (!vmMo.createSnapshot(templateUniqueName, "Temporary snapshot for template creation", false, false)) {
                  String msg = "Unable to take snapshot for creating template from volume. volume path: " + volumePath;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              // 4 MB is the minimum requirement for VM memory in VMware
              String vmxFormattedVirtualHardwareVersion = VirtualMachineMO.getVmxFormattedVirtualHardwareVersion(vmMo.getVirtualHardwareVersion());
              vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), vmxFormattedVirtualHardwareVersion);
              clonedVm = vmMo.getRunningHost().findVmOnHyperHost(workerVmName);
              if (clonedVm == null) {
                  String msg = "Unable to create dummy VM to export volume. volume path: " + volumePath;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              clonedVm.exportVm(secondaryMountPoint + "/" + installPath, templateUniqueName, true, false);
  
              long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length();
              OVAProcessor processor = new OVAProcessor();
              Map<String, Object> params = new HashMap<String, Object>();
              params.put(StorageLayer.InstanceConfigKey, _storage);
              processor.configure("OVA Processor", params);
              long virtualSize = processor.getTemplateVirtualSize(installFullPath, templateUniqueName);
  
              postCreatePrivateTemplate(installFullPath, templateId, templateUniqueName, physicalSize, virtualSize);
              return new Ternary<String, Long, Long>(installPath + "/" + templateUniqueName + ".ova", physicalSize, virtualSize);
  
          } finally {
              if (clonedVm != null) {
                  clonedVm.detachAllDisksAndDestroy();
              }
  
              vmMo.removeSnapshot(templateUniqueName, false);
          }
      }
  
      private Ternary<String, Long, Long> createTemplateFromSnapshot(long accountId, long templateId, String templateUniqueName, String secStorageUrl, long volumeId,
                                                                     String backedUpSnapshotUuid, String nfsVersion) throws Exception {
  
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsVersion);
          String installPath = getTemplateRelativeDirInSecStorage(accountId, templateId);
          String installFullPath = secondaryMountPoint + "/" + installPath;
          String installFullOVAName = installFullPath + "/" + templateUniqueName + ".ova";  //Note: volss for tmpl
          String snapshotRoot = secondaryMountPoint + "/" + getSnapshotRelativeDirInSecStorage(accountId, volumeId);
          String snapshotFullOVAName = snapshotRoot + "/" + backedUpSnapshotUuid + ".ova";
          String snapshotFullOvfName = snapshotRoot + "/" + backedUpSnapshotUuid + ".ovf";
          String result;
          Script command;
          String templateVMDKName = "";
          //String snapshotFullVMDKName = snapshotRoot + "/";
          // the backedUpSnapshotUuid field currently has the format: uuid/uuid. so we need to extract the uuid out
          String backupSSUuid = backedUpSnapshotUuid.substring(0, backedUpSnapshotUuid.indexOf('/'));
          String snapshotFullVMDKName = snapshotRoot + "/" + backupSSUuid + "/";
  
          synchronized (installPath.intern()) {
              command = new Script(false, "mkdir", _timeout, logger);
              command.add("-p");
              command.add(installFullPath);
  
              result = command.execute();
              if (result != null) {
                  String msg = "unable to prepare template directory: " + installPath + ", storage: " + secStorageUrl + ", error msg: " + result;
                  logger.error(msg);
                  throw new Exception(msg);
              }
          }
  
          try {
              if (new File(snapshotFullOVAName).exists()) {
                  command = new Script(false, "cp", _timeout, logger);
                  command.add(snapshotFullOVAName);
                  command.add(installFullOVAName);
                  result = command.execute();
                  if (result != null) {
                      String msg = "unable to copy snapshot " + snapshotFullOVAName + " to " + installFullPath;
                      logger.error(msg);
                      throw new Exception(msg);
                  }
  
                  // untar OVA file at template directory
                  command = new Script("tar", 0, logger);
                  command.add("--no-same-owner");
                  command.add("-xf", installFullOVAName);
                  command.setWorkDir(installFullPath);
                  logger.info("Executing command: " + command.toString());
                  result = command.execute();
                  if (result != null) {
                      String msg = "unable to untar snapshot " + snapshotFullOVAName + " to " + installFullPath;
                      logger.error(msg);
                      throw new Exception(msg);
                  }
  
              } else {  // there is no ova file, only ovf originally;
                  if (new File(snapshotFullOvfName).exists()) {
                      command = new Script(false, "cp", _timeout, logger);
                      command.add(snapshotFullOvfName);
                      //command.add(installFullOvfName);
                      command.add(installFullPath);
                      result = command.execute();
                      if (result != null) {
                          String msg = "unable to copy snapshot " + snapshotFullOvfName + " to " + installFullPath;
                          logger.error(msg);
                          throw new Exception(msg);
                      }
  
                      logger.info("vmdkfile parent dir: " + snapshotFullVMDKName);
                      File snapshotdir = new File(snapshotFullVMDKName);
                      // File snapshotdir = new File(snapshotRoot);
                      File[] ssfiles = snapshotdir.listFiles();
                      // List<String> filenames = new ArrayList<String>();
                      for (int i = 0; i < ssfiles.length; i++) {
                          String vmdkfile = ssfiles[i].getName();
                          logger.info("vmdk file name: " + vmdkfile);
                          if (vmdkfile.toLowerCase().startsWith(backupSSUuid) && vmdkfile.toLowerCase().endsWith(".vmdk")) {
                              snapshotFullVMDKName += vmdkfile;
                              templateVMDKName += vmdkfile;
                              break;
                          }
                      }
                      if (snapshotFullVMDKName != null) {
                          command = new Script(false, "cp", _timeout, logger);
                          command.add(snapshotFullVMDKName);
                          command.add(installFullPath);
                          result = command.execute();
                          logger.info("Copy VMDK file: " + snapshotFullVMDKName);
                          if (result != null) {
                              String msg = "unable to copy snapshot vmdk file " + snapshotFullVMDKName + " to " + installFullPath;
                              logger.error(msg);
                              throw new Exception(msg);
                          }
                      }
                  } else {
                      String msg = "unable to find any snapshot ova/ovf files" + snapshotFullOVAName + " to " + installFullPath;
                      logger.error(msg);
                      throw new Exception(msg);
                  }
              }
  
              long physicalSize = new File(installFullPath + "/" + templateVMDKName).length();
              OVAProcessor processor = new OVAProcessor();
              // long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length();
              Map<String, Object> params = new HashMap<String, Object>();
              params.put(StorageLayer.InstanceConfigKey, _storage);
              processor.configure("OVA Processor", params);
              long virtualSize = processor.getTemplateVirtualSize(installFullPath, templateUniqueName);
  
              postCreatePrivateTemplate(installFullPath, templateId, templateUniqueName, physicalSize, virtualSize);
              writeMetaOvaForTemplate(installFullPath, backedUpSnapshotUuid + ".ovf", templateVMDKName, templateUniqueName, physicalSize);
              return new Ternary<String, Long, Long>(installPath + "/" + templateUniqueName + ".ova", physicalSize, virtualSize);
          } catch (Exception e) {
              // TODO, clean up left over files
              throw e;
          }
      }
  
      private void postCreatePrivateTemplate(String installFullPath, long templateId, String templateName, long size, long virtualSize) throws Exception {
  
          // TODO a bit ugly here
          BufferedWriter out = null;
          try {
              out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(installFullPath + "/template.properties"), "UTF-8"));
              out.write("filename=" + templateName + ".ova");
              out.newLine();
              out.write("description=");
              out.newLine();
              out.write("checksum=");
              out.newLine();
              out.write("hvm=false");
              out.newLine();
              out.write("size=" + size);
              out.newLine();
              //out.write("ova=true");
              out.write("ova=false");  //volss: the real ova file is not created
              out.newLine();
              out.write("id=" + templateId);
              out.newLine();
              out.write("public=false");
              out.newLine();
              out.write("ova.filename=" + templateName + ".ova");
              out.newLine();
              out.write("uniquename=" + templateName);
              out.newLine();
              out.write("ova.virtualsize=" + virtualSize);
              out.newLine();
              out.write("virtualsize=" + virtualSize);
              out.newLine();
              out.write("ova.size=" + size);
              out.newLine();
          } finally {
              if (out != null) {
                  out.close();
              }
          }
      }
  
      private void writeMetaOvaForTemplate(String installFullPath, String ovfFilename, String vmdkFilename, String templateName, long diskSize) throws Exception {
  
          // TODO a bit ugly here
          BufferedWriter out = null;
          try {
              out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(installFullPath + "/" + templateName + ".ova.meta"), "UTF-8"));
              out.write("ova.filename=" + templateName + ".ova");
              out.newLine();
              out.write("version=1.0");
              out.newLine();
              out.write("ovf=" + ovfFilename);
              out.newLine();
              out.write("numDisks=1");
              out.newLine();
              out.write("disk1.name=" + vmdkFilename);
              out.newLine();
              out.write("disk1.size=" + diskSize);
              out.newLine();
          } finally {
              if (out != null) {
                  out.close();
              }
          }
      }
  
      private String createVolumeFromSnapshot(VmwareHypervisorHost hyperHost, DatastoreMO primaryDsMo, String newVolumeName, long accountId, long volumeId, String secStorageUrl,
                                              String snapshotBackupUuid, String nfsVersion) throws Exception {
  
          restoreVolumeFromSecStorage(hyperHost, primaryDsMo, newVolumeName, secStorageUrl, getSnapshotRelativeDirInSecStorage(accountId, volumeId), snapshotBackupUuid, nfsVersion);
          return null;
      }
  
      private void restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, DatastoreMO primaryDsMo, String newVolumeName, String secStorageUrl, String secStorageDir,
                                               String backupName, String nfsVersion) throws Exception {
  
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsVersion);
          String srcOVAFileName = secondaryMountPoint + "/" + secStorageDir + "/" + backupName + "." + ImageFormat.OVA.getFileExtension();
          String snapshotDir = "";
          if (backupName.contains("/")) {
              snapshotDir = backupName.split("/")[0];
          }
  
          File ovafile = new File(srcOVAFileName);
          String srcOVFFileName = secondaryMountPoint + "/" + secStorageDir + "/" + backupName + ".ovf";
          File ovfFile = new File(srcOVFFileName);
          // String srcFileName = getOVFFilePath(srcOVAFileName);
          if (!ovfFile.exists()) {
              srcOVFFileName = getOVFFilePath(srcOVAFileName);
              if (srcOVFFileName == null && ovafile.exists()) {  // volss: ova file exists; o/w can't do tar
                  Script command = new Script("tar", 0, logger);
                  command.add("--no-same-owner");
                  command.add("-xf", srcOVAFileName);
                  command.setWorkDir(secondaryMountPoint + "/" + secStorageDir + "/" + snapshotDir);
                  logger.info("Executing command: " + command.toString());
                  String result = command.execute();
                  if (result != null) {
                      String msg = "Unable to unpack snapshot OVA file at: " + srcOVAFileName;
                      logger.error(msg);
                      throw new Exception(msg);
                  }
              } else {
                  String msg = "Unable to find snapshot OVA file at: " + srcOVAFileName;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              srcOVFFileName = getOVFFilePath(srcOVAFileName);
          }
          if (srcOVFFileName == null) {
              String msg = "Unable to locate OVF file in template package directory: " + srcOVAFileName;
              logger.error(msg);
              throw new Exception(msg);
          }
  
          VirtualMachineMO clonedVm = null;
          try {
              hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin", null);
              clonedVm = hyperHost.findVmOnHyperHost(newVolumeName);
              if (clonedVm == null) {
                  throw new Exception("Unable to create container VM for volume creation");
              }
  
              clonedVm.moveAllVmDiskFiles(primaryDsMo, "", false);
              clonedVm.detachAllDisks();
          } finally {
              if (clonedVm != null) {
                  clonedVm.detachAllDisksAndDestroy();
              }
          }
      }
  
      private String backupSnapshotToSecondaryStorage(VirtualMachineMO vmMo, long accountId, long volumeId, String volumePath, String snapshotUuid, String secStorageUrl,
                                                      String prevSnapshotUuid, String prevBackupUuid, String workerVmName, String nfsVersion) throws Exception {
  
          String backupUuid = UUID.randomUUID().toString();
          exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, getSnapshotRelativeDirInSecStorage(accountId, volumeId), backupUuid, workerVmName, nfsVersion, true);
          return backupUuid + "/" + backupUuid;
      }
  
      private void exportVolumeToSecondaryStorage(VirtualMachineMO vmMo, String volumePath, String secStorageUrl, String secStorageDir, String exportName, String workerVmName,
                                                  String nfsVersion, boolean clonedWorkerVMNeeded) throws Exception {
  
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsVersion);
          String exportPath = secondaryMountPoint + "/" + secStorageDir + "/" + exportName;
  
          synchronized (exportPath.intern()) {
              if (!new File(exportPath).exists()) {
                  Script command = new Script(false, "mkdir", _timeout, logger);
                  command.add("-p");
                  command.add(exportPath);
  
                  String result = command.execute();
                  if (result != null) {
                      String errorMessage = String.format("Unable to prepare snapshot backup directory: [%s] due to [%s].", exportPath, result);
                      logger.error(errorMessage);
                      throw new Exception(errorMessage);
                  }
              }
          }
  
          VirtualMachineMO clonedVm = null;
          try {
  
              Pair<VirtualDisk, String> volumeDeviceInfo = vmMo.getDiskDevice(volumePath);
              if (volumeDeviceInfo == null) {
                  String msg = "Unable to find related disk device for volume. volume path: " + volumePath;
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              if (clonedWorkerVMNeeded) {
                  // 4 MB is the minimum requirement for VM memory in VMware
                  String vmxFormattedVirtualHardwareVersion = VirtualMachineMO.getVmxFormattedVirtualHardwareVersion(vmMo.getVirtualHardwareVersion());
                  vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), vmxFormattedVirtualHardwareVersion);
                  clonedVm = vmMo.getRunningHost().findVmOnHyperHost(workerVmName);
                  if (clonedVm == null) {
                      String msg = String.format("Unable to create dummy VM to export volume. volume path: [%s].", volumePath);
                      logger.error(msg);
                      throw new Exception(msg);
                  }
                  clonedVm.exportVm(exportPath, exportName, false, false);  //Note: volss: not to create ova.
              } else {
                  vmMo.exportVm(exportPath, exportName, false, false);
              }
          } finally {
              if (clonedVm != null) {
                  clonedVm.detachAllDisksAndDestroy();
              }
          }
      }
  
      private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostService, VmwareHypervisorHost hyperHost, CopyVolumeCommand cmd, String vmName, long volumeId,
                                                          String poolId, String volumePath, String secStorageUrl, String workerVmName, String nfsVersion) throws Exception {
  
          String volumeFolder = String.valueOf(volumeId) + "/";
          VirtualMachineMO workerVm = null;
          VirtualMachineMO vmMo = null;
          String exportName = UUID.randomUUID().toString();
          String searchExcludedFolders = cmd.getContextParam("searchexludefolders");
  
  
          try {
              ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId);
  
              if (morDs == null) {
                  String msg = "Unable to find volumes's storage pool for copy volume operation";
                  logger.error(msg);
                  throw new Exception(msg);
              }
  
              boolean clonedWorkerVMNeeded = true;
              vmMo = hyperHost.findVmOnHyperHost(vmName);
              if (vmMo == null) {
                  // create a dummy worker vm for attaching the volume
                  DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
                  workerVm = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName, null);
  
                  if (workerVm == null) {
                      String msg = "Unable to create worker VM to execute CopyVolumeCommand";
                      logger.error(msg);
                      throw new Exception(msg);
                  }
  
                  //attach volume to worker VM
                  String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk", searchExcludedFolders);
                  workerVm.attachDisk(new String[] {datastoreVolumePath}, morDs);
                  vmMo = workerVm;
                  clonedWorkerVMNeeded = false;
              } else {
                  vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false);
              }
  
              exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1, null),
                      nfsVersion, clonedWorkerVMNeeded);
              return new Pair<String, String>(volumeFolder, exportName);
  
          } finally {
              if (vmMo != null && vmMo.getSnapshotMor(exportName) != null) {
                  vmMo.removeSnapshot(exportName, false);
              }
              if (workerVm != null) {
                  workerVm.detachAllDisksAndDestroy();
              }
          }
      }
  
      private String getVolumePathInDatastore(DatastoreMO dsMo, String volumeFileName, String searchExcludeFolders) throws Exception {
          String datastoreVolumePath = dsMo.searchFileInSubFolders(volumeFileName, true, searchExcludeFolders);
          if (datastoreVolumePath == null) {
              throw new CloudRuntimeException("Unable to find file " + volumeFileName + " in datastore " + dsMo.getName());
          }
          return datastoreVolumePath;
      }
  
      private Pair<String, String> copyVolumeFromSecStorage(VmwareHypervisorHost hyperHost, long volumeId, DatastoreMO dsMo, String secStorageUrl, String exportName,
                                                            String nfsVersion) throws Exception {
  
          String volumeFolder = String.valueOf(volumeId) + "/";
          String newVolume = UUID.randomUUID().toString().replace("-", "");
          restoreVolumeFromSecStorage(hyperHost, dsMo, newVolume, secStorageUrl, "volumes/" + volumeFolder, exportName, nfsVersion);
  
          return new Pair<String, String>(volumeFolder, newVolume);
      }
  
      // here we use a method to return the ovf and vmdk file names; Another way to do it:
      // create a new class, and like TemplateLocation.java and create templateOvfInfo.java to handle it;
      private String createOVAFromMetafile(String metafileName, int archiveTimeout) throws Exception {
          File ova_metafile = new File(metafileName);
          Properties props = null;
          String ovaFileName = "";
          logger.info("Creating OVA using MetaFile: " + metafileName);
          try (FileInputStream strm = new FileInputStream(ova_metafile);) {
  
              logger.info("loading properties from ova meta file: " + metafileName);
              props = new Properties();
              props.load(strm);
              ovaFileName = props.getProperty("ova.filename");
              logger.info("ovafilename: " + ovaFileName);
              String ovfFileName = props.getProperty("ovf");
              logger.info("ovffilename: " + ovfFileName);
              int diskNum = Integer.parseInt(props.getProperty("numDisks"));
              if (diskNum <= 0) {
                  String msg = "VMDK disk file number is 0. Error";
                  logger.error(msg);
                  throw new Exception(msg);
              }
              String[] disks = new String[diskNum];
              for (int i = 0; i < diskNum; i++) {
                  // String diskNameKey = "disk" + Integer.toString(i+1) + ".name"; // Fang use this
                  String diskNameKey = "disk1.name";
                  disks[i] = props.getProperty(diskNameKey);
                  logger.info("diskname " + disks[i]);
              }
              String exportDir = ova_metafile.getParent();
              logger.info("exportDir: " + exportDir);
              // Important! we need to sync file system before we can safely use tar to work around a linux kernel bug(or feature)
              logger.info("Sync file system before we package OVA..., before tar ");
              logger.info("ova: " + ovaFileName + ", ovf:" + ovfFileName + ", vmdk:" + disks[0] + ".");
              Script commandSync = new Script(true, "sync", 0, logger);
              commandSync.execute();
              Script command = new Script(false, "tar", archiveTimeout, logger);
              command.setWorkDir(exportDir); // Fang: pass this in to the method?
              command.add("-cf", ovaFileName);
              command.add(ovfFileName); // OVF file should be the first file in OVA archive
              for (String diskName : disks) {
                  command.add(diskName);
              }
              command.execute();
              logger.info("Package OVA for template in dir: " + exportDir + "cmd: " + command.toString());
              // to be safe, physically test existence of the target OVA file
              if ((new File(exportDir + File.separator + ovaFileName)).exists()) {
                  logger.info("OVA file: " + ovaFileName + " is created and ready to extract.");
                  return ovaFileName;
              } else {
                  String msg = exportDir + File.separator + ovaFileName + " is not created as expected";
                  logger.error(msg);
                  throw new Exception(msg);
              }
          } catch (Exception e) {
              logger.error("Exception while creating OVA using Metafile", e);
              throw e;
          }
  
      }
  
      private String getOVFFilePath(String srcOVAFileName) {
          File file = new File(srcOVAFileName);
          assert (_storage != null);
          String[] files = _storage.listFiles(file.getParent());
          if (files != null) {
              for (String fileName : files) {
                  if (fileName.toLowerCase().endsWith(".ovf")) {
                      File ovfFile = new File(fileName);
                      return file.getParent() + File.separator + ovfFile.getName();
                  }
              }
          }
          return null;
      }
  
      private static String getTemplateRelativeDirInSecStorage(long accountId, long templateId) {
          return "template/tmpl/" + accountId + "/" + templateId;
      }
  
      private static String getSnapshotRelativeDirInSecStorage(long accountId, long volumeId) {
          return "snapshots/" + accountId + "/" + volumeId;
      }
  
      protected boolean isManagedStorageDatastorePath(final String datastorePath) {
          // ex. [-iqn.2010-01.com.solidfire:3p53.data-9999.97-0] i-2-9999-VM
          return datastorePath != null && datastorePath.startsWith("[-iqn.");
      }
  
      protected String getManagedDatastoreName(final String datastorePath) {
          // ex. [-iqn.2010-01.com.solidfire:3p53.data-9999.97-0]
          return datastorePath == null ? datastorePath : datastorePath.split(" ")[0];
      }
  
      private long getVMSnapshotChainSize(VmwareContext context, VmwareHypervisorHost hyperHost, String fileName, ManagedObjectReference morDs,
                                          String exceptFileName, String vmName) throws Exception {
          long size = 0;
          DatastoreMO dsMo = new DatastoreMO(context, morDs);
          HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO();
          String datastorePath = (new DatastoreFile(dsMo.getName(), vmName)).getPath();
          if (isManagedStorageDatastorePath(datastorePath)) {
              datastorePath = getManagedDatastoreName(datastorePath);
          }
          HostDatastoreBrowserSearchSpec searchSpec = new HostDatastoreBrowserSearchSpec();
          FileQueryFlags fqf = new FileQueryFlags();
          fqf.setFileSize(true);
          fqf.setFileOwner(true);
          fqf.setModification(true);
          searchSpec.setDetails(fqf);
          searchSpec.setSearchCaseInsensitive(false);
          searchSpec.getMatchPattern().add(fileName);
          ArrayList<HostDatastoreBrowserSearchResults> results = browserMo.searchDatastoreSubFolders(datastorePath, searchSpec);
          for (HostDatastoreBrowserSearchResults result : results) {
              if (result != null) {
                  List<FileInfo> info = result.getFile();
                  for (FileInfo fi : info) {
                      if (exceptFileName != null && fi.getPath().contains(exceptFileName)) {
                          continue;
                      } else {
                          size = size + fi.getFileSize();
                      }
                  }
              }
          }
          return size;
      }
  
      private boolean isVolumeOnDatastoreCluster(VolumeObjectTO volumeObjectTO) {
          DataStoreTO dsTO = volumeObjectTO.getDataStore();
          if (!(dsTO instanceof PrimaryDataStoreTO)) {
              return false;
          }
          PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)dsTO;
          return Storage.StoragePoolType.DatastoreCluster.equals(primaryDataStoreTO.getPoolType()) ||
                  Storage.StoragePoolType.DatastoreCluster.equals(primaryDataStoreTO.getParentPoolType());
      }
  
      private void syncVolume(VmwareHostService hostService, VirtualMachineMO virtualMachineMO, VmwareContext context,
                               VmwareHypervisorHost hypervisorHost, VolumeObjectTO volumeTO) throws Exception {
          if (hostService.getStorageProcessor() == null) return;
          VmwareStorageProcessor storageProcessor = hostService.getStorageProcessor();
          DiskTO disk = new DiskTO();
          Map<String, String> map = new HashMap<>();
          map.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString());
          disk.setDetails(map);
          disk.setData(volumeTO);
          storageProcessor.getSyncedVolume(virtualMachineMO, context, hypervisorHost, disk, volumeTO);
      }
  
      @Override
      public CreateVMSnapshotAnswer execute(VmwareHostService hostService, CreateVMSnapshotCommand cmd) {
          List<VolumeObjectTO> volumeTOs = cmd.getVolumeTOs();
          String vmName = cmd.getVmName();
          String vmSnapshotName = cmd.getTarget().getSnapshotName();
          String vmSnapshotDesc = cmd.getTarget().getDescription();
          boolean snapshotMemory = cmd.getTarget().getType() == VMSnapshot.Type.DiskAndMemory;
          boolean quiescevm = cmd.getTarget().getQuiescevm();
          VirtualMachineMO vmMo = null;
          VmwareContext context = hostService.getServiceContext(cmd);
  
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
  
              // wait if there are already VM snapshot task running
              ManagedObjectReference taskmgr = context.getServiceContent().getTaskManager();
              List<ManagedObjectReference> tasks = context.getVimClient().getDynamicProperty(taskmgr, "recentTask");
  
              for (ManagedObjectReference taskMor : tasks) {
                  TaskInfo info = (TaskInfo)(context.getVimClient().getDynamicProperty(taskMor, "info"));
  
                  if (info.getEntityName().equals(cmd.getVmName()) && org.apache.commons.lang3.StringUtils.isNotBlank(info.getName()) && info.getName().equalsIgnoreCase("CreateSnapshot_Task")) {
                      if (!(info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR))) {
                          logger.debug("There is already a VM snapshot task running, wait for it");
                          context.getVimClient().waitForTask(taskMor);
                      }
                  }
              }
  
              vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              }
  
              if (vmMo == null) {
                  String msg = "Unable to find VM for CreateVMSnapshotCommand";
                  logger.info(msg);
  
                  return new CreateVMSnapshotAnswer(cmd, false, msg);
              } else {
                  if (vmMo.getSnapshotMor(vmSnapshotName) != null) {
                      logger.info("VM snapshot " + vmSnapshotName + " already exists");
                  } else if (!vmMo.createSnapshot(vmSnapshotName, vmSnapshotDesc, snapshotMemory, quiescevm)) {
                      return new CreateVMSnapshotAnswer(cmd, false, "Unable to create snapshot due to esxi internal failed");
                  }
  
                  setVolumeToPathAndSize(volumeTOs, vmMo, hostService, context, hyperHost);
  
                  return new CreateVMSnapshotAnswer(cmd, cmd.getTarget(), volumeTOs);
              }
          } catch (Exception e) {
              String msg = e.getMessage();
              logger.error("failed to create snapshot for vm:" + vmName + " due to " + msg, e);
  
              try {
                  if (vmMo.getSnapshotMor(vmSnapshotName) != null) {
                      vmMo.removeSnapshot(vmSnapshotName, false);
                  }
              } catch (Exception e1) {
                  logger.info("[ignored]" + "error during snapshot remove: " + e1.getLocalizedMessage());
              }
  
              return new CreateVMSnapshotAnswer(cmd, false, e.getMessage());
          }
      }
  
      private Map<String, String> getNewDiskMap(VirtualMachineMO vmMo) throws Exception {
          Map<String, String> mapNewDisk = new HashMap<String, String>();
  
          // find VM disk file path after creating snapshot
          VirtualDisk[] vdisks = vmMo.getAllDiskDevice();
  
          for (int i = 0; i < vdisks.length; i++) {
              List<Pair<String, ManagedObjectReference>> vmdkFiles = vmMo.getDiskDatastorePathChain(vdisks[i], false);
  
              for (Pair<String, ManagedObjectReference> fileItem : vmdkFiles) {
                  String fullPath = fileItem.first();
                  String baseName = null;
                  String vmdkName = null;
  
                  // if this is managed storage
                  if (isManagedStorageDatastorePath(fullPath)) {
                      baseName = getManagedDatastoreName(fullPath);
                      baseName = baseName.substring(1, baseName.length() - 1); // remove '[' and ']'
  
                      vmdkName = fullPath; // for managed storage, vmdkName == fullPath
                  } else {
                      vmdkName = fullPath.split("] ")[1];
  
                      if (vmdkName.endsWith(".vmdk")) {
                          vmdkName = vmdkName.substring(0, vmdkName.length() - (".vmdk").length());
                      }
  
                      String token = "/";
  
                      if (vmdkName.contains(token)) {
                          vmdkName = vmdkName.substring(vmdkName.indexOf(token) + token.length());
                      }
  
                      baseName = VmwareHelper.trimSnapshotDeltaPostfix(vmdkName);
                  }
  
                  mapNewDisk.put(baseName, vmdkName);
              }
          }
  
          return mapNewDisk;
      }
  
      protected void setVolumeToPathAndSize(List<VolumeObjectTO> volumeTOs, VirtualMachineMO vmMo, VmwareHostService hostService, VmwareContext context, VmwareHypervisorHost hyperHost)
              throws Exception {
          String vmName = vmMo.getVmName();
          for (VolumeObjectTO volumeTO : volumeTOs) {
              String path = volumeTO.getPath();
              String baseName;
              String datastoreUuid = volumeTO.getDataStore().getUuid();
  
              if (isVolumeOnDatastoreCluster(volumeTO)) {
                  syncVolume(hostService, vmMo, context, hyperHost, volumeTO);
                  path = volumeTO.getPath();
                  baseName = VmwareHelper.trimSnapshotDeltaPostfix(volumeTO.getPath());
                  if (StringUtils.isNotEmpty(volumeTO.getDataStoreUuid())) {
                      datastoreUuid = volumeTO.getDataStoreUuid();
                  }
              } else {
                  Map<String, String> mapNewDisk = getNewDiskMap(vmMo);
                  if (isManagedStorageDatastorePath(path)) {
                      path = getManagedDatastoreName(path);
                      baseName = path.substring(1, path.length() - 1); // remove '[' and ']'
                  } else {
                      baseName = VmwareHelper.trimSnapshotDeltaPostfix(path);
                  }
                  path = mapNewDisk.get(baseName);
                  volumeTO.setPath(path);
              }
  
              // get volume's chain size for this VM snapshot; exclude current volume vdisk
              ManagedObjectReference morDs = getDatastoreAsManagedObjectReference(baseName, hyperHost, datastoreUuid);
              long size = getVMSnapshotChainSize(context, hyperHost, baseName + "-*.vmdk", morDs, path, vmName);
  
              if (volumeTO.getVolumeType() == Volume.Type.ROOT) {
                  // add memory snapshot size
                  size += getVMSnapshotChainSize(context, hyperHost, vmName + "-*.vmsn", morDs, null, vmName);
              }
  
              volumeTO.setSize(size);
          }
      }
  
      private ManagedObjectReference getDatastoreAsManagedObjectReference(String baseName, VmwareHypervisorHost hyperHost, String storeUuid) throws Exception {
          try {
              // if baseName equates to a datastore name, this should be managed storage
              ManagedObjectReference morDs = hyperHost.findDatastoreByName(baseName);
  
              if (morDs != null) {
                  return morDs;
              }
          } catch (Exception ex) {
              logger.info("[ignored]" + "error getting managed object refference: " + ex.getLocalizedMessage());
          }
  
          // not managed storage, so use the standard way of getting a ManagedObjectReference for a datastore
          return HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, storeUuid);
      }
  
      @Override
      public DeleteVMSnapshotAnswer execute(VmwareHostService hostService, DeleteVMSnapshotCommand cmd) {
          List<VolumeObjectTO> listVolumeTo = cmd.getVolumeTOs();
          VirtualMachineMO vmMo = null;
          VmwareContext context = hostService.getServiceContext(cmd);
          String vmName = cmd.getVmName();
          String vmSnapshotName = cmd.getTarget().getSnapshotName();
  
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
              vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              }
  
              if (vmMo == null) {
                  String msg = "Unable to find VM for RevertToVMSnapshotCommand";
                  logger.debug(msg);
  
                  return new DeleteVMSnapshotAnswer(cmd, false, msg);
              } else {
                  if (vmMo.getSnapshotMor(vmSnapshotName) == null) {
                      logger.debug("can not find the snapshot " + vmSnapshotName + ", assume it is already removed");
                  } else {
                      if (!vmMo.removeSnapshot(vmSnapshotName, false)) {
                          String msg = "delete vm snapshot " + vmSnapshotName + " due to error occurred in vmware";
                          logger.error(msg);
  
                          return new DeleteVMSnapshotAnswer(cmd, false, msg);
                      }
                  }
  
                  logger.debug("snapshot: " + vmSnapshotName + " is removed");
  
                  // after removed snapshot, the volumes' paths have been changed for the VM, needs to report new paths to manager
  
                  setVolumeToPathAndSize(listVolumeTo, vmMo, hostService, context, hyperHost);
  
                  return new DeleteVMSnapshotAnswer(cmd, listVolumeTo);
              }
          } catch (Exception e) {
              String msg = e.getMessage();
              logger.error("failed to delete vm snapshot " + vmSnapshotName + " of vm " + vmName + " due to " + msg, e);
  
              return new DeleteVMSnapshotAnswer(cmd, false, msg);
          }
      }
  
      @Override
      public RevertToVMSnapshotAnswer execute(VmwareHostService hostService, RevertToVMSnapshotCommand cmd) {
          String snapshotName = cmd.getTarget().getSnapshotName();
          String vmName = cmd.getVmName();
          Boolean snapshotMemory = cmd.getTarget().getType() == VMSnapshot.Type.DiskAndMemory;
          List<VolumeObjectTO> listVolumeTo = cmd.getVolumeTOs();
          VirtualMachine.PowerState vmState = VirtualMachine.PowerState.PowerOn;
          VirtualMachineMO vmMo = null;
          VmwareContext context = hostService.getServiceContext(cmd);
  
          try {
              VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
  
              // wait if there are already VM revert task running
              ManagedObjectReference taskmgr = context.getServiceContent().getTaskManager();
              List<ManagedObjectReference> tasks = context.getVimClient().getDynamicProperty(taskmgr, "recentTask");
  
              for (ManagedObjectReference taskMor : tasks) {
                  TaskInfo info = (TaskInfo)(context.getVimClient().getDynamicProperty(taskMor, "info"));
  
                  if (info.getEntityName().equals(cmd.getVmName()) && org.apache.commons.lang3.StringUtils.isNotBlank(info.getName()) && info.getName().equalsIgnoreCase("RevertToSnapshot_Task")) {
                      logger.debug("There is already a VM snapshot task running, wait for it");
                      context.getVimClient().waitForTask(taskMor);
                  }
              }
  
              HostMO hostMo = (HostMO)hyperHost;
              vmMo = hyperHost.findVmOnHyperHost(vmName);
  
              if (vmMo == null) {
                  vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
              }
  
              if (vmMo == null) {
                  String msg = "Unable to find VM for RevertToVMSnapshotCommand";
                  logger.debug(msg);
  
                  return new RevertToVMSnapshotAnswer(cmd, false, msg);
              } else {
                  if (cmd.isReloadVm()) {
                      vmMo.reload();
                  }
  
                  boolean result = false;
  
                  if (snapshotName != null) {
                      ManagedObjectReference morSnapshot = vmMo.getSnapshotMor(snapshotName);
  
                      result = hostMo.revertToSnapshot(morSnapshot);
                  } else {
                      return new RevertToVMSnapshotAnswer(cmd, false, "Unable to find the snapshot by name " + snapshotName);
                  }
  
                  if (result) {
                      setVolumeToPathAndSize(listVolumeTo, vmMo, hostService, context, hyperHost);
  
                      if (!snapshotMemory) {
                          vmState = VirtualMachine.PowerState.PowerOff;
                      }
  
                      return new RevertToVMSnapshotAnswer(cmd, listVolumeTo, vmState);
                  } else {
                      return new RevertToVMSnapshotAnswer(cmd, false, "Error while reverting to snapshot due to execute in ESXi");
                  }
              }
          } catch (Exception e) {
              String msg = "revert vm " + vmName + " to snapshot " + snapshotName + " failed due to " + e.getMessage();
              logger.error(msg, e);
  
              return new RevertToVMSnapshotAnswer(cmd, false, msg);
          }
      }
  
      private String deleteVolumeDirOnSecondaryStorage(long volumeId, String secStorageUrl, String nfsVersion) throws Exception {
          String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl, nfsVersion);
          String volumeMountRoot = secondaryMountPoint + "/" + getVolumeRelativeDirInSecStorage(volumeId);
  
          return deleteDir(volumeMountRoot);
      }
  
      private String deleteDir(String dir) {
          synchronized (dir.intern()) {
              Script command = new Script(false, "rm", _timeout, logger);
              command.add("-rf");
              command.add(dir);
              return command.execute();
          }
      }
  
      private static String getVolumeRelativeDirInSecStorage(long volumeId) {
          return "volumes/" + volumeId;
      }
  }
  