content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.hypervisor.kvm.storage;
  
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  
  import org.apache.cloudstack.utils.qemu.QemuImg;
  import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
  import org.apache.cloudstack.utils.qemu.QemuImgException;
  import org.apache.cloudstack.utils.qemu.QemuImgFile;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.logging.log4j.Logger;
  import org.apache.logging.log4j.LogManager;
  import org.libvirt.LibvirtException;
  
  import com.cloud.agent.api.to.DiskTO;
  import com.cloud.storage.Storage;
  import com.cloud.storage.Storage.ProvisioningType;
  import com.cloud.storage.Storage.StoragePoolType;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.script.OutputInterpreter;
  import com.cloud.utils.script.Script;
  
  public class IscsiAdmStorageAdaptor implements StorageAdaptor {
      protected Logger logger = LogManager.getLogger(getClass());
  
      private static final Map<String, KVMStoragePool> MapStorageUuidToStoragePool = new HashMap<>();
  
      @Override
      public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map<String, String> details, boolean isPrimaryStorage) {
          IscsiAdmStoragePool storagePool = new IscsiAdmStoragePool(uuid, host, port, storagePoolType, this);
  
          MapStorageUuidToStoragePool.put(uuid, storagePool);
  
          return storagePool;
      }
  
      @Override
      public StoragePoolType getStoragePoolType() {
          return StoragePoolType.Iscsi;
      }
  
      @Override
      public KVMStoragePool getStoragePool(String uuid) {
          return MapStorageUuidToStoragePool.get(uuid);
      }
  
      @Override
      public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
          return MapStorageUuidToStoragePool.get(uuid);
      }
  
      @Override
      public boolean deleteStoragePool(String uuid) {
          return MapStorageUuidToStoragePool.remove(uuid) != null;
      }
  
      @Override
      public boolean deleteStoragePool(KVMStoragePool pool) {
          return deleteStoragePool(pool.getUuid());
      }
  
      // called from LibvirtComputingResource.execute(CreateCommand)
      // does not apply for iScsiAdmStorageAdaptor
      @Override
      public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
          throw new UnsupportedOperationException("Creating a physical disk is not supported.");
      }
  
      @Override
      public boolean connectPhysicalDisk(String volumeUuid, KVMStoragePool pool, Map<String, String> details, boolean isVMMigrate) {
          // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 -o new
          Script iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
  
          iScsiAdmCmd.add("-m", "node");
          iScsiAdmCmd.add("-T", getIqn(volumeUuid));
          iScsiAdmCmd.add("-p", pool.getSourceHost() + ":" + pool.getSourcePort());
          iScsiAdmCmd.add("-o", "new");
  
          String result = iScsiAdmCmd.execute();
  
          if (result != null) {
              logger.debug("Failed to add iSCSI target " + volumeUuid);
              System.out.println("Failed to add iSCSI target " + volumeUuid);
  
              return false;
          } else {
              logger.debug("Successfully added iSCSI target " + volumeUuid);
              System.out.println("Successfully added to iSCSI target " + volumeUuid);
          }
  
          String chapInitiatorUsername = details.get(DiskTO.CHAP_INITIATOR_USERNAME);
          String chapInitiatorSecret = details.get(DiskTO.CHAP_INITIATOR_SECRET);
  
          if (StringUtils.isNoneBlank(chapInitiatorUsername, chapInitiatorSecret)) {
              try {
                  // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --op update -n node.session.auth.authmethod -v CHAP
                  executeChapCommand(volumeUuid, pool, "node.session.auth.authmethod", "CHAP", null);
  
                  // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --op update -n node.session.auth.username -v username
                  executeChapCommand(volumeUuid, pool, "node.session.auth.username", chapInitiatorUsername, "username");
  
                  // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --op update -n node.session.auth.password -v password
                  executeChapCommand(volumeUuid, pool, "node.session.auth.password", chapInitiatorSecret, "password");
              } catch (Exception ex) {
                  return false;
              }
          }
  
          // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --login
          iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
  
          iScsiAdmCmd.add("-m", "node");
          iScsiAdmCmd.add("-T", getIqn(volumeUuid));
          iScsiAdmCmd.add("-p", pool.getSourceHost() + ":" + pool.getSourcePort());
          iScsiAdmCmd.add("--login");
  
          result = iScsiAdmCmd.execute();
  
          if (result != null) {
              logger.debug("Failed to log in to iSCSI target " + volumeUuid);
              System.out.println("Failed to log in to iSCSI target " + volumeUuid);
  
              return false;
          } else {
              logger.debug("Successfully logged in to iSCSI target " + volumeUuid);
              System.out.println("Successfully logged in to iSCSI target " + volumeUuid);
          }
  
          // There appears to be a race condition where logging in to the iSCSI volume via iscsiadm
          // returns success before the device has been added to the OS.
          // What happens is you get logged in and the device shows up, but the device may not
          // show up before we invoke Libvirt to attach the device to a VM.
          // waitForDiskToBecomeAvailable(String, KVMStoragePool) invokes blockdev
          // via getPhysicalDisk(String, KVMStoragePool) and checks if the size came back greater
          // than 0.
          // After a certain number of tries and a certain waiting period in between tries,
          // this method could still return (it should not block indefinitely) (the race condition
          // isn't solved here, but made highly unlikely to be a problem).
          waitForDiskToBecomeAvailable(volumeUuid, pool);
  
          return true;
      }
  
      private void waitForDiskToBecomeAvailable(String volumeUuid, KVMStoragePool pool) {
          int numberOfTries = 10;
          int timeBetweenTries = 1000;
  
          while (getPhysicalDisk(volumeUuid, pool).getSize() == 0 && numberOfTries > 0) {
              numberOfTries--;
  
              try {
                  Thread.sleep(timeBetweenTries);
              } catch (Exception ex) {
                  // don't do anything
              }
          }
      }
  
      private void waitForDiskToBecomeUnavailable(String host, int port, String iqn, String lun) {
          int numberOfTries = 10;
          int timeBetweenTries = 1000;
  
          String deviceByPath = getByPath(host, port, "/" + iqn + "/" + lun);
  
          while (getDeviceSize(deviceByPath) > 0 && numberOfTries > 0) {
              numberOfTries--;
  
              try {
                  Thread.sleep(timeBetweenTries);
              } catch (Exception ex) {
                  // don't do anything
              }
          }
      }
  
      private void executeChapCommand(String path, KVMStoragePool pool, String nParameter, String vParameter, String detail) throws Exception {
          Script iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
  
          iScsiAdmCmd.add("-m", "node");
          iScsiAdmCmd.add("-T", getIqn(path));
          iScsiAdmCmd.add("-p", pool.getSourceHost() + ":" + pool.getSourcePort());
          iScsiAdmCmd.add("--op", "update");
          iScsiAdmCmd.add("-n", nParameter);
          iScsiAdmCmd.add("-v", vParameter);
  
          String result = iScsiAdmCmd.execute();
  
          boolean useDetail = detail != null && detail.trim().length() > 0;
  
          detail = useDetail ? detail.trim() + " " : detail;
  
          if (result != null) {
              logger.debug("Failed to execute CHAP " + (useDetail ? detail : "") + "command for iSCSI target " + path + " : message = " + result);
              System.out.println("Failed to execute CHAP " + (useDetail ? detail : "") + "command for iSCSI target " + path + " : message = " + result);
  
              throw new Exception("Failed to execute CHAP " + (useDetail ? detail : "") + "command for iSCSI target " + path + " : message = " + result);
          } else {
              logger.debug("CHAP " + (useDetail ? detail : "") + "command executed successfully for iSCSI target " + path);
              System.out.println("CHAP " + (useDetail ? detail : "") + "command executed successfully for iSCSI target " + path);
          }
      }
  
      // example by-path: /dev/disk/by-path/ip-192.168.233.10:3260-iscsi-iqn.2012-03.com.solidfire:storagepool2-lun-0
      private String getByPath(String host, int port, String path) {
          return "/dev/disk/by-path/ip-" + host + ":" + port + "-iscsi-" + getIqn(path) + "-lun-" + getLun(path);
      }
  
      @Override
      public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
          String deviceByPath = getByPath(pool.getSourceHost(), pool.getSourcePort(), volumeUuid);
          KVMPhysicalDisk physicalDisk = new KVMPhysicalDisk(deviceByPath, volumeUuid, pool);
  
          physicalDisk.setFormat(PhysicalDiskFormat.RAW);
  
          long deviceSize = getDeviceSize(deviceByPath);
  
          physicalDisk.setSize(deviceSize);
          physicalDisk.setVirtualSize(deviceSize);
  
          return physicalDisk;
      }
  
      private long getDeviceSize(String deviceByPath) {
          Script iScsiAdmCmd = new Script(true, "blockdev", 0, logger);
  
          iScsiAdmCmd.add("--getsize64", deviceByPath);
  
          OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
  
          String result = iScsiAdmCmd.execute(parser);
  
          if (result != null) {
              logger.warn("Unable to retrieve the size of device (resource may have moved to a different host)" + deviceByPath);
  
              return 0;
          }
          else {
              logger.info("Successfully retrieved the size of device " + deviceByPath);
          }
  
          return Long.parseLong(parser.getLine());
      }
  
      private String getIqn(String path) {
          return getComponent(path, 1);
      }
  
      private String getLun(String path) {
          return getComponent(path, 2);
      }
  
      private String getComponent(String path, int index) {
          String[] tmp = path.split("/");
  
          if (tmp.length != 3) {
              String msg = "Wrong format for iScsi path: " + path + ". It should be formatted as '/targetIQN/LUN'.";
  
              logger.warn(msg);
  
              throw new CloudRuntimeException(msg);
          }
  
          return tmp[index].trim();
      }
  
      private boolean disconnectPhysicalDisk(String host, int port, String iqn, String lun) {
          // use iscsiadm to log out of the iSCSI target and un-discover it
  
          // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 --logout
          Script iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
  
          iScsiAdmCmd.add("-m", "node");
          iScsiAdmCmd.add("-T", iqn);
          iScsiAdmCmd.add("-p", host + ":" + port);
          iScsiAdmCmd.add("--logout");
  
          String result = iScsiAdmCmd.execute();
  
          if (result != null) {
              logger.debug("Failed to log out of iSCSI target /" + iqn + "/" + lun + " : message = " + result);
              System.out.println("Failed to log out of iSCSI target /" + iqn + "/" + lun + " : message = " + result);
  
              return false;
          } else {
              logger.debug("Successfully logged out of iSCSI target /" + iqn + "/" + lun);
              System.out.println("Successfully logged out of iSCSI target /" + iqn + "/" + lun);
          }
  
          // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 -o delete
          iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
  
          iScsiAdmCmd.add("-m", "node");
          iScsiAdmCmd.add("-T", iqn);
          iScsiAdmCmd.add("-p", host + ":" + port);
          iScsiAdmCmd.add("-o", "delete");
  
          result = iScsiAdmCmd.execute();
  
          if (result != null) {
              logger.debug("Failed to remove iSCSI target /" + iqn + "/" + lun + " : message = " + result);
              System.out.println("Failed to remove iSCSI target /" + iqn + "/" + lun + " : message = " + result);
  
              return false;
          } else {
              logger.debug("Removed iSCSI target /" + iqn + "/" + lun);
              System.out.println("Removed iSCSI target /" + iqn + "/" + lun);
          }
  
          waitForDiskToBecomeUnavailable(host, port, iqn, lun);
  
          return true;
      }
  
      @Override
      public boolean disconnectPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
          return disconnectPhysicalDisk(pool.getSourceHost(), pool.getSourcePort(), getIqn(volumeUuid), getLun(volumeUuid));
      }
  
      @Override
      public boolean disconnectPhysicalDisk(Map<String, String> volumeToDisconnect) {
          String host = volumeToDisconnect.get(DiskTO.STORAGE_HOST);
          String port = volumeToDisconnect.get(DiskTO.STORAGE_PORT);
          String path = volumeToDisconnect.get(DiskTO.IQN);
  
          if (host != null && port != null && path != null) {
              return disconnectPhysicalDisk(host, Integer.parseInt(port), getIqn(path), getLun(path));
          }
  
          return false;
      }
  
      @Override
      public boolean disconnectPhysicalDiskByPath(String localPath) {
          String search1 = "/dev/disk/by-path/ip-";
          String search2 = ":";
          String search3 = "-iscsi-";
          String search4 = "-lun-";
  
          if (!localPath.contains(search3)) {
              return false;
          }
  
          int index = localPath.indexOf(search2);
  
          String host = localPath.substring(search1.length(), index);
  
          int index2 = localPath.indexOf(search3);
  
          String port = localPath.substring(index + search2.length(), index2);
  
          index = localPath.indexOf(search4);
  
          String iqn = localPath.substring(index2 + search3.length(), index);
  
          String lun = localPath.substring(index + search4.length());
  
          return disconnectPhysicalDisk(host, Integer.parseInt(port), iqn, lun);
      }
  
      @Override
      public boolean deletePhysicalDisk(String volumeUuid, KVMStoragePool pool, Storage.ImageFormat format) {
          throw new UnsupportedOperationException("Deleting a physical disk is not supported.");
      }
  
      // does not apply for iScsiAdmStorageAdaptor
      @Override
      public List<KVMPhysicalDisk> listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) {
          throw new UnsupportedOperationException("Listing disks is not supported for this configuration.");
      }
  
      @Override
      public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format,
              ProvisioningType provisioningType, long size,
              KVMStoragePool destPool, int timeout, byte[] passphrase) {
          throw new UnsupportedOperationException("Creating a disk from a template is not yet supported for this configuration.");
      }
  
      @Override
      public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) {
          throw new UnsupportedOperationException("Creating a template from a disk is not yet supported for this configuration.");
      }
  
      @Override
      public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) {
          return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null);
      }
  
      @Override
      public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolumeUuid, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] destPassphrase, ProvisioningType provisioningType) {
  
          QemuImgFile srcFile;
  
          KVMStoragePool srcPool = srcDisk.getPool();
  
          if (srcPool.getType() == StoragePoolType.RBD) {
              srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat());
          } else {
              srcFile = new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat());
          }
  
          KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(destVolumeUuid);
  
          QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat());
  
          try {
              QemuImg q = new QemuImg(timeout);
              q.convert(srcFile, destFile);
          } catch (QemuImgException | LibvirtException ex) {
              String msg = "Failed to copy data from " + srcDisk.getPath() + " to " +
                      destDisk.getPath() + ". The error was the following: " + ex.getMessage();
  
              logger.error(msg);
  
              throw new CloudRuntimeException(msg);
          }
  
          return destPool.getPhysicalDisk(destVolumeUuid);
      }
  
      @Override
      public boolean refresh(KVMStoragePool pool) {
          return true;
      }
  
      @Override
      public boolean createFolder(String uuid, String path) {
          return createFolder(uuid, path, null);
      }
  
      @Override
      public boolean createFolder(String uuid, String path, String localPath) {
          throw new UnsupportedOperationException("A folder cannot be created in this configuration.");
      }
  
      @Override
      public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) {
          return null;
      }
  
      @Override
      public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) {
          return null;
      }
  }
  