content: //
  // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  //
  
  package org.apache.cloudstack.direct.download;
  
  import java.io.ByteArrayInputStream;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.OutputStream;
  import java.net.URL;
  import java.nio.charset.StandardCharsets;
  import java.security.KeyManagementException;
  import java.security.KeyStore;
  import java.security.KeyStoreException;
  import java.security.NoSuchAlgorithmException;
  import java.security.cert.CertificateException;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Map;
  
  import javax.net.ssl.HttpsURLConnection;
  import javax.net.ssl.SSLContext;
  
  import org.apache.commons.collections.MapUtils;
  import org.apache.commons.httpclient.HttpStatus;
  import org.apache.commons.io.IOUtils;
  import org.apache.http.Header;
  import org.apache.http.HttpEntity;
  import org.apache.http.client.config.RequestConfig;
  import org.apache.http.client.methods.CloseableHttpResponse;
  import org.apache.http.client.methods.HttpGet;
  import org.apache.http.client.methods.HttpHead;
  import org.apache.http.client.methods.HttpUriRequest;
  import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
  import org.apache.http.impl.client.CloseableHttpClient;
  import org.apache.http.impl.client.HttpClients;
  import org.apache.http.ssl.SSLContexts;
  import org.apache.http.util.EntityUtils;
  
  import com.cloud.utils.Pair;
  import com.cloud.utils.UriUtils;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.script.Script;
  import com.cloud.utils.storage.QCOW2Utils;
  
  public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
  
      protected CloseableHttpClient httpsClient;
      private HttpUriRequest req;
  
      protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
          this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects);
      }
  
      public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
                   Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
                   Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
          super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
          SSLContext sslcontext = getSSLContext();
          SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
          RequestConfig config = RequestConfig.custom()
                  .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
                  .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
                  .setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
                  .setRedirectsEnabled(followRedirects)
                  .build();
          httpsClient = HttpClients.custom()
                  .setSSLSocketFactory(factory)
                  .setDefaultRequestConfig(config)
                  .build();
          createUriRequest(url, headers);
          String downloadDir = getDirectDownloadTempPath(templateId);
          File tempFile = createTemporaryDirectoryAndFile(downloadDir);
          setDownloadedFilePath(tempFile.getAbsolutePath());
      }
  
      protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
          req = new HttpGet(downloadUrl);
          setFollowRedirects(this.isFollowRedirects());
          if (MapUtils.isNotEmpty(headers)) {
              for (Map.Entry<String, String> entry : headers.entrySet()) {
                  req.setHeader(entry.getKey(), entry.getValue());
              }
          }
      }
  
      private SSLContext getSSLContext() {
          try {
              KeyStore customKeystore  = KeyStore.getInstance("jks");
              try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) {
                  String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null  | sed 's/keystore.passphrase=//g' 2>/dev/null";
                  String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
                  String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
                  customKeystore.load(instream, privatePassword.toCharArray());
              }
              KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType());
              String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator);
              String filename = System.getProperty("java.home") + relativeCacertsPath;
              try (FileInputStream is = new FileInputStream(filename)) {
                  String password = "changeit";
                  defaultKeystore.load(is, password.toCharArray());
              }
              return SSLContexts.custom()
                      .loadTrustMaterial(customKeystore, null)
                      .loadTrustMaterial(defaultKeystore, null)
                      .build();
          } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
              logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e);
              try {
                  return SSLContext.getDefault();
              } catch (NoSuchAlgorithmException ex) {
                  throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e);
              }
          }
  
      }
  
      @Override
      public Pair<Boolean, String> downloadTemplate() {
          CloseableHttpResponse response;
          try {
              response = httpsClient.execute(req);
          } catch (IOException e) {
              throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
          }
          return consumeResponse(response);
      }
  
      /**
       * Consume response and persist it on getDownloadedFilePath() file
       */
      protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
          logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
          if (response.getStatusLine().getStatusCode() != 200) {
              throw new CloudRuntimeException("Error on HTTPS response");
          }
          try {
              HttpEntity entity = response.getEntity();
              InputStream in = entity.getContent();
              OutputStream out = new FileOutputStream(getDownloadedFilePath());
              IOUtils.copy(in, out);
          } catch (Exception e) {
              logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
              return new Pair<>(false, null);
          }
          return new Pair<>(true, getDownloadedFilePath());
      }
  
      @Override
      public boolean checkUrl(String url) {
          HttpHead httpHead = new HttpHead(url);
          try {
              CloseableHttpResponse response = httpsClient.execute(httpHead);
              int responseCode = response.getStatusLine().getStatusCode();
              if (responseCode != HttpStatus.SC_OK) {
                  logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
                  return false;
              }
              return true;
          } catch (IOException e) {
              logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e);
              return false;
          } finally {
              httpHead.releaseConnection();
          }
      }
  
      @Override
      public Long getRemoteFileSize(String url, String format) {
          if ("qcow2".equalsIgnoreCase(format)) {
              try {
                  URL urlObj = new URL(url);
                  HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection();
                  SSLContext context = getSSLContext();
                  urlConnection.setSSLSocketFactory(context.getSocketFactory());
                  urlConnection.connect();
                  return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url));
              } catch (IOException e) {
                  throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
              }
          } else {
              HttpHead httpHead = new HttpHead(url);
              CloseableHttpResponse response = null;
              try {
                  response = httpsClient.execute(httpHead);
                  Header[] headers = response.getHeaders("Content-Length");
                  for (Header header : headers) {
                      return Long.parseLong(header.getValue());
                  }
              } catch (IOException e) {
                  throw new RuntimeException(e);
              }
              return null;
          }
      }
  
      @Override
      public List<String> getMetalinkUrls(String metalinkUrl) {
          HttpGet getMethod = new HttpGet(metalinkUrl);
          List<String> urls = new ArrayList<>();
          CloseableHttpResponse response;
          try {
              response = httpsClient.execute(getMethod);
              if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
                  String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl);
                  logger.error(msg);
                  throw new IOException(msg);
              }
          } catch (IOException e) {
              logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e);
              getMethod.releaseConnection();
              return null;
          }
  
          try {
              String responseStr = EntityUtils.toString(response.getEntity());
              ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8));
              addMetalinkUrlsToListFromInputStream(inputStream, urls);
          } catch (IOException e) {
              logger.warn(e.getMessage(), e);
          } finally {
              getMethod.releaseConnection();
          }
          return urls;
      }
  
      @Override
      public List<String> getMetalinkChecksums(String metalinkUrl) {
          HttpGet getMethod = new HttpGet(metalinkUrl);
          try {
              CloseableHttpResponse response = httpsClient.execute(getMethod);
              if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                  InputStream is = response.getEntity().getContent();
                  return generateChecksumListFromInputStream(is);
              }
          } catch (IOException e) {
              logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e);
          } finally {
              getMethod.releaseConnection();
          }
          return null;
      }
  }
  