content: /**
   * ﻿Copyright (C) 2007 - 2019 52°North Initiative for Geospatial Open Source
   * Software GmbH
   *
   * This program is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 as published
   * by the Free Software Foundation.
   *
   * If the program is linked with libraries which are licensed under one of
   * the following licenses, the combination of the program with the linked
   * library is not considered a "derivative work" of the program:
   *
   *       • Apache License, version 2.0
   *       • Apache Software License, version 1.0
   *       • GNU Lesser General Public License, version 3
   *       • Mozilla Public License, versions 1.0, 1.1 and 2.0
   *       • Common Development and Distribution License (CDDL), version 1.0
   *
   * Therefore the distribution of the program linked with libraries licensed
   * under the aforementioned licenses, is permitted by the copyright holders
   * if the distribution is compliant with both the GNU General Public
   * License version 2 and the aforementioned licenses.
   *
   * This program is distributed in the hope that it will be useful, but
   * WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
   * Public License for more details.
   */
  package org.n52.wps.server.database;
  
  import java.io.BufferedOutputStream;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.OutputStream;
  import java.sql.Connection;
  import java.text.SimpleDateFormat;
  import java.util.Date;
  import java.util.Timer;
  import java.util.TimerTask;
  import java.util.UUID;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  import java.util.zip.GZIPInputStream;
  import java.util.zip.GZIPOutputStream;
  
  import org.apache.commons.io.IOUtils;
  import org.n52.wps.DatabaseDocument.Database;
  import org.n52.wps.ServerDocument.Server;
  import org.n52.wps.commons.MIMEUtil;
  import org.n52.wps.commons.PropertyUtil;
  import org.n52.wps.commons.WPSConfig;
  import org.n52.wps.commons.XMLUtil;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import com.google.common.base.Joiner;
  
  /*
   * @author tkunicki (Thomas Kunicki, USGS)
   *
   */
  public final class FlatFileDatabase extends AbstractDatabase {
  
      private final static Logger LOGGER = LoggerFactory.getLogger(FlatFileDatabase.class);
  
      private final static String KEY_DATABASE_ROOT = "org.n52.wps.server.database";
      private final static String KEY_DATABASE_PATH = "path";
      private final static String KEY_DATABASE_WIPE_ENABLED = "wipe.enabled";
      private final static String KEY_DATABASE_WIPE_PERIOD = "wipe.period";
      private final static String KEY_DATABASE_WIPE_THRESHOLD = "wipe.threshold";
      private final static String KEY_DATABASE_COMPLEX_GZIP = "complex.gzip";
      
      private final static String DEFAULT_DATABASE_PATH = 
              Joiner.on(File.separator).join(
                  System.getProperty("java.io.tmpdir", "."),
                  "Database",
                  "Results");
      private final static boolean DEFAULT_DATABASE_WIPE_ENABLED = true;
      private final static long DEFAULT_DATABASE_WIPE_PERIOD = 1000 * 60 * 60;  // P1H
      private final static long DEFAULT_DATABASE_WIPE_THRESHOLD = 1000 * 60 * 60 * 24 * 7; // P7D
      private final static boolean DEFAULT_DATABASE_COMPLEX_GZIP = true; // P7D
      
      private final static String SUFFIX_MIMETYPE = "mime-type";
      private final static String SUFFIX_CONTENT_LENGTH = "content-length";
      private final static String SUFFIX_XML = "xml";
      private final static String SUFFIX_TEMP = "tmp";
      private final static String SUFFIX_GZIP = "gz";
      private final static String SUFFIX_PROPERTIES = "properties";
  
      // If the delimiter changes, examine Patterns below.
      private final static Joiner JOINER = Joiner.on(".");
  
      // Grouping is used to pull out integer index of response, if these patterns
      // change examine findLatestResponseIndex(...), generateResponseFile(...)
      // and generateResponseFile(...)
      private final static Pattern PATTERN_RESPONSE = Pattern.compile("([\\d]+)\\." + SUFFIX_XML);
      private final static Pattern PATTERN_RESPONSE_TEMP = Pattern.compile("([\\d]+)\\." + SUFFIX_XML + "(:?\\."
              + SUFFIX_TEMP + ")?");
  
      private static FlatFileDatabase instance;
  
      // This method is required by the DatabaseFactory, it is found using reflection
      public synchronized static IDatabase getInstance() {
          if (instance == null) {
              instance = new FlatFileDatabase();
          }
          return instance;
      }
  
      protected final File baseDirectory;
  
      protected final boolean gzipComplexValues;
  
      protected final Object storeResponseSerialNumberLock;
  
      protected final boolean indentXML = true;
  
      protected final Timer wipeTimer;
  
      protected FlatFileDatabase() {
          
          Server server = WPSConfig.getInstance().getWPSConfig().getServer();
          Database database = server.getDatabase();
          PropertyUtil propertyUtil = new PropertyUtil(database.getPropertyArray(), KEY_DATABASE_ROOT);
          LOGGER.info("Using \"{}\" as base URL for results", getBaseResultURL());
          
          String baseDirectoryPath = propertyUtil.extractString(KEY_DATABASE_PATH, DEFAULT_DATABASE_PATH);
          baseDirectory = new File(baseDirectoryPath);
          LOGGER.info("Using \"{}\" as base directory for results database", baseDirectoryPath);
          if ( !baseDirectory.exists()) {
              LOGGER.info("Results database does not exist, creating.", baseDirectoryPath);
              baseDirectory.mkdirs();
          }
  
          if (propertyUtil.extractBoolean(KEY_DATABASE_WIPE_ENABLED, DEFAULT_DATABASE_WIPE_ENABLED)) {
              
              long periodMillis = propertyUtil.extractPeriodAsMillis(KEY_DATABASE_WIPE_PERIOD, DEFAULT_DATABASE_WIPE_PERIOD);
              long thresholdMillis = propertyUtil.extractPeriodAsMillis(KEY_DATABASE_WIPE_THRESHOLD, DEFAULT_DATABASE_WIPE_THRESHOLD);
  
              wipeTimer = new Timer(getClass().getSimpleName() + " File Wiper", true);
              wipeTimer.scheduleAtFixedRate(new FlatFileDatabase.WipeTimerTask(thresholdMillis), 0, periodMillis);
              LOGGER.info("Started {} file wiper timer; period {} ms, threshold {} ms",
                      new Object[] {getDatabaseName(),periodMillis,thresholdMillis});
          } else {
              wipeTimer = null;
          }
  
          gzipComplexValues = propertyUtil.extractBoolean(KEY_DATABASE_COMPLEX_GZIP, DEFAULT_DATABASE_COMPLEX_GZIP);
  
          storeResponseSerialNumberLock = new Object();
      }
  
      @Override
      public String getDatabaseName() {
          return getClass().getSimpleName();
      }
  
      @Override
      public void insertRequest(String id, InputStream inputStream, boolean xml) {
          // store request in response directory...
          File responseDirectory = generateResponseDirectory(id);
          responseDirectory.mkdir();
          BufferedOutputStream outputStream = null;
          try {
              if (xml) {
                  outputStream = new BufferedOutputStream(
                          new FileOutputStream(
                              new File(
                                  responseDirectory,
                                  JOINER.join("request", SUFFIX_XML)),
                          false));
                  XMLUtil.copyXML(inputStream, outputStream, indentXML);
              } else {
                  outputStream = new BufferedOutputStream(
                          new FileOutputStream(
                              new File(
                                  responseDirectory,
                                  JOINER.join("request", SUFFIX_PROPERTIES)),
                          false));
                  IOUtils.copy(inputStream, outputStream);
              }
          }
          catch (Exception e) {
              LOGGER.error("Exception storing request for id {}: {}", id, e);
          }
          finally {
              IOUtils.closeQuietly(inputStream);
              IOUtils.closeQuietly(outputStream);
          }
      }
  
      @Override
      public String insertResponse(String id, InputStream outputStream) {
          return this.storeResponse(id, outputStream);
      }
  
      @Override
      public InputStream lookupRequest(String id) {
          File requestFile = lookupRequestAsFile(id);
          if (requestFile != null && requestFile.exists()) {
              LOGGER.debug("Request file for {} is {}", id, requestFile.getPath());
              try {
                  return new FileInputStream(requestFile);
              }
              catch (FileNotFoundException ex) {
                  // should never get here due to checks above...
                  LOGGER.warn("Request not found for id {}", id);
              }
          }
          LOGGER.warn("Response not found for id {}", id);
          return null;
      }
  
      @Override
      public InputStream lookupResponse(String id) {
          File responseFile = lookupResponseAsFile(id);
          if (responseFile != null && responseFile.exists()) {
              LOGGER.debug("Response file for {} is {}", id, responseFile.getPath());
              try {
                  return responseFile.getName().endsWith(SUFFIX_GZIP) ? new GZIPInputStream(new FileInputStream(responseFile))
                                                                     : new FileInputStream(responseFile);
              }
              catch (FileNotFoundException ex) {
                  // should never get here due to checks above...
                  LOGGER.warn("Response not found for id {}", id);
              }
              catch (IOException ex) {
                  LOGGER.warn("Error processing response for id {}", id);
              }
          }
          LOGGER.warn("Response not found for id {}", id);
          return null;
      }
  
      @Override
      public File lookupRequestAsFile(String id) {
          File requestAsFile = null;
          // request is stored in response directory...
          File responseDirectory = generateResponseDirectory(id);
          if (responseDirectory.exists()) {
              synchronized (storeResponseSerialNumberLock) {
                  requestAsFile = new File(responseDirectory, JOINER.join("request", SUFFIX_XML));
                  if ( !requestAsFile.exists()) {
                      requestAsFile = new File(responseDirectory, JOINER.join("request", SUFFIX_PROPERTIES));
                  }
                  if ( !requestAsFile.exists()) {
                      requestAsFile = null;
                  }
              }
          }
          return requestAsFile;
      }
  
      @Override
      public File lookupResponseAsFile(String id) {
          File responseFile = null;
          // if response resolved to directory, this means the response is a status update
          File responseDirectory = generateResponseDirectory(id);
          if (responseDirectory.exists()) {
              synchronized (storeResponseSerialNumberLock) {
                  return findLatestResponseFile(responseDirectory);
              }
          }
          else {
              String mimeType = getMimeTypeForStoreResponse(id);
              if (mimeType != null) {
                  // ignore gzipComplexValues in case file was stored when value
                  // was inconsistent with current value;
                  responseFile = generateComplexDataFile(id, mimeType, false);
                  if ( !responseFile.exists()) {
                      responseFile = generateComplexDataFile(id, mimeType, true);
                  }
                  if ( !responseFile.exists()) {
                      responseFile = null;
                  }
              }
          }
          return responseFile;
      }
  
      @Override
      public void shutdown() {
          if (wipeTimer != null) {
              wipeTimer.cancel();
          }
      }
  
      @Override
      public String storeComplexValue(String id, InputStream resultInputStream, String type, String mimeType) {
  
          String resultId = JOINER.join(id, UUID.randomUUID().toString());
          try {
              File resultFile = generateComplexDataFile(resultId, mimeType, gzipComplexValues);
              File mimeTypeFile = generateComplexDataMimeTypeFile(resultId);
              File contentLengthFile = generateComplexDataContentLengthFile(resultId);
  
              LOGGER.debug("initiating storage of complex value for {} as {}", id, resultFile.getPath());
  
              long contentLength = -1;
  
              OutputStream resultOutputStream = null;
              try {
                  resultOutputStream = gzipComplexValues ? new GZIPOutputStream(new FileOutputStream(resultFile))
                                                        : new BufferedOutputStream(new FileOutputStream(resultFile));
                  contentLength = IOUtils.copyLarge(resultInputStream, resultOutputStream);
              }
              finally {
                  IOUtils.closeQuietly(resultInputStream);
                  IOUtils.closeQuietly(resultOutputStream);
              }
  
              OutputStream mimeTypeOutputStream = null;
              try {
                  mimeTypeOutputStream = new BufferedOutputStream(new FileOutputStream(mimeTypeFile));
                  IOUtils.write(mimeType, mimeTypeOutputStream);
              }
              finally {
                  IOUtils.closeQuietly(mimeTypeOutputStream);
              }
  
              OutputStream contentLengthOutputStream = null;
              try {
                  contentLengthOutputStream = new BufferedOutputStream(new FileOutputStream(contentLengthFile));
                  IOUtils.write(Long.toString(contentLength), contentLengthOutputStream);
              }
              finally {
                  IOUtils.closeQuietly(contentLengthOutputStream);
              }
  
              LOGGER.debug("completed storage of complex value for {} as {}", id, resultFile.getPath());
  
          }
          catch (IOException e) {
              throw new RuntimeException("Error storing complex value for " + resultId, e);
          }
          return generateRetrieveResultURL(resultId);
      }
  
      @Override
      public String storeResponse(String id, InputStream inputStream) {
  
          try {
              File responseTempFile;
              File responseFile;
              synchronized (storeResponseSerialNumberLock) {
                  File responseDirectory = generateResponseDirectory(id);
                  responseDirectory.mkdir();
                  int responseIndex = findLatestResponseIndex(responseDirectory, true);
                  if (responseIndex < 0) {
                      responseIndex = 0;
                  }
                  else {
                      responseIndex++;
                  }
                  responseFile = generateResponseFile(responseDirectory, responseIndex);
                  responseTempFile = generateResponseTempFile(responseDirectory, responseIndex);
                  try {
                      // create the file so that the reponse serial number is correctly
                      // incremented if this method is called again for this reponse
                      // before this reponse is completed.
                      responseTempFile.createNewFile();
                  }
                  catch (IOException e) {
                      throw new RuntimeException("Error storing response to {}", e);
                  }
                  LOGGER.debug("Creating temp file for {} as {}", id, responseTempFile.getPath());
              }
              InputStream responseInputStream = null;
              OutputStream responseOutputStream = null;
              try {
                  responseInputStream = inputStream;
                  responseOutputStream = new BufferedOutputStream(new FileOutputStream(responseTempFile));
                  // In order to allow the prior response to be available we write
                  // to a temp file and rename these when completed. Large responses
                  // can cause the call below to take a significant amount of time.
                  XMLUtil.copyXML(responseInputStream, responseOutputStream, indentXML);
              }
              finally {
                  IOUtils.closeQuietly(responseInputStream);
                  IOUtils.closeQuietly(responseOutputStream);
              }
  
              synchronized (storeResponseSerialNumberLock) {
                  responseTempFile.renameTo(responseFile);
                  LOGGER.debug("Renamed temp file for {} to {}", id, responseFile.getPath());
              }
  
              return generateRetrieveResultURL(id);
  
          }
          catch (FileNotFoundException e) {
              throw new RuntimeException("Error storing response for " + id, e);
          }
          catch (IOException e) {
              throw new RuntimeException("Error storing response for " + id, e);
          }
      }
  
      @Override
      public void updateResponse(String id, InputStream inputStream) {
          this.storeResponse(id, inputStream);
      }
  
      @Override
      public String getMimeTypeForStoreResponse(String id) {
  
          File responseDirectory = generateResponseDirectory(id);
          if (responseDirectory.exists()) {
              return "text/xml";
          }
          else {
              File mimeTypeFile = generateComplexDataMimeTypeFile(id);
              if (mimeTypeFile.canRead()) {
                  InputStream mimeTypeInputStream = null;
                  try {
                      mimeTypeInputStream = new FileInputStream(mimeTypeFile);
                      return IOUtils.toString(mimeTypeInputStream);
                  }
                  catch (IOException e) {
                      throw new RuntimeException(e);
                  }
                  finally {
                      IOUtils.closeQuietly(mimeTypeInputStream);
                  }
              }
          }
          return null;
      }
  
      @Override
      public long getContentLengthForStoreResponse(String id) {
  
          File responseDirectory = generateResponseDirectory(id);
          if (responseDirectory.exists()) {
              synchronized (storeResponseSerialNumberLock) {
                  File responseFile = findLatestResponseFile(responseDirectory);
                  return responseFile.length();
              }
          }
          else {
              File contentLengthFile = generateComplexDataContentLengthFile(id);
              if (contentLengthFile.canRead()) {
                  InputStream contentLengthInputStream = null;
                  try {
                      contentLengthInputStream = new FileInputStream(contentLengthFile);
                      return Long.parseLong(IOUtils.toString(contentLengthInputStream));
                  }
                  catch (IOException e) {
                      LOGGER.error("Unable to extract content-length for response id {} from {}, exception message: {}",
                                   new Object[] {id, contentLengthFile.getAbsolutePath(), e.getMessage()});
                  }
                  catch (NumberFormatException e) {
                      LOGGER.error("Unable to parse content-length for response id {} from {}, exception message: {}",
                                   new Object[] {id, contentLengthFile.getAbsolutePath(), e.getMessage()});
                  }
                  finally {
                      IOUtils.closeQuietly(contentLengthInputStream);
                  }
              }
              return -1;
          }
      }
  
      @Override
      public boolean deleteStoredResponse(String id) {
          return false;
      }
  
      private int findLatestResponseIndex(File responseDirectory, boolean includeTemp) {
          int responseIndex = Integer.MIN_VALUE;
          for (File file : responseDirectory.listFiles()) {
              Matcher matcher = includeTemp ? PATTERN_RESPONSE_TEMP.matcher(file.getName())
                                           : PATTERN_RESPONSE.matcher(file.getName());
              if (matcher.matches()) {
                  int fileIndex = Integer.parseInt(matcher.group(1));
                  if (fileIndex > responseIndex) {
                      responseIndex = fileIndex;
                  }
              }
          }
          return responseIndex;
      }
  
      private File findLatestResponseFile(File responseDirectory) {
          int responseIndex = findLatestResponseIndex(responseDirectory, false);
          return responseIndex < 0 ? null : generateResponseFile(responseDirectory, responseIndex);
      }
  
      private File generateResponseFile(File responseDirectory, int index) {
          return new File(responseDirectory, JOINER.join(index, SUFFIX_XML));
      }
  
      private File generateResponseTempFile(File responseDirectory, int index) {
          return new File(responseDirectory, JOINER.join(index, SUFFIX_XML, SUFFIX_TEMP));
      }
  
      private File generateResponseDirectory(String id) {
          return new File(baseDirectory, id);
      }
  
      private File generateComplexDataFile(String id, String mimeType, boolean gzip) {
          String fileName = gzip ? JOINER.join(id, MIMEUtil.getSuffixFromMIMEType(mimeType), SUFFIX_GZIP)
                                : JOINER.join(id, MIMEUtil.getSuffixFromMIMEType(mimeType));
          return new File(baseDirectory, fileName);
      }
  
      private File generateComplexDataMimeTypeFile(String id) {
          return new File(baseDirectory, JOINER.join(id, SUFFIX_MIMETYPE));
      }
  
      private File generateComplexDataContentLengthFile(String id) {
          return new File(baseDirectory, JOINER.join(id, SUFFIX_CONTENT_LENGTH));
      }
  
      private class WipeTimerTask extends TimerTask {
  
          public final long thresholdMillis;
  
          WipeTimerTask(long thresholdMillis) {
              this.thresholdMillis = thresholdMillis;
          }
  
          @Override
          public void run() {
              wipe(baseDirectory, thresholdMillis);
          }
  
          private void wipe(File rootFile, long thresholdMillis) {
              // SimpleDataFormat is not thread-safe.
              SimpleDateFormat iso8601DateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
              long currentTimeMillis = System.currentTimeMillis();
              LOGGER.info(getDatabaseName() + " file wiper, checking {} for files older than {} ms",
                          rootFile.getAbsolutePath(),
                          thresholdMillis);
  
              File[] files = rootFile.listFiles();
              if (files != null) {
                  for (File file : files) {
                      long lastModifiedMillis = file.lastModified();
                      long ageMillis = currentTimeMillis - lastModifiedMillis;
                      if (ageMillis > thresholdMillis) {
                          LOGGER.info("Deleting {}, last modified date is {}",
                                      file.getName(),
                                      iso8601DateFormat.format(new Date(lastModifiedMillis)));
                          delete(file);
                          if (file.exists()) {
                              LOGGER.warn("Deletion of {} failed", file.getName());
                          }
                      }
                  }
              } else {
                  LOGGER.warn("Cannot delete files, no files in root directory {}  > file list is null. ", rootFile.getAbsolutePath());
              }
          }
  
          private void delete(File file) {
              if (file.isDirectory()) {
                  for (File child : file.listFiles()) {
                      delete(child);
                  }
              }
              file.delete();
          }
      }
  
      @Override
      public Connection getConnection() {
          return null;
      }
  
      @Override
      public String getConnectionURL() {
          return null;
      }
  }
  