content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package act.installer;
  
  import act.installer.pubchem.PubchemParser;
  import act.server.MongoDB;
  import act.shared.Chemical;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.DefaultParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.commons.lang3.tuple.Triple;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  import org.jaxen.JaxenException;
  import org.jaxen.XPath;
  import org.jaxen.dom.DOMXPath;
  import org.json.JSONArray;
  import org.json.JSONException;
  import org.json.JSONObject;
  import org.w3c.dom.Document;
  import org.w3c.dom.Node;
  import org.xml.sax.SAXException;
  
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.DocumentBuilderFactory;
  import javax.xml.parsers.ParserConfigurationException;
  import java.io.File;
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.SortedSet;
  import java.util.TreeSet;
  import java.util.regex.Pattern;
  import java.util.stream.Collectors;
  
  public class HMDBParser {
    private static final Logger LOGGER = LogManager.getFormatterLogger(HMDBParser.class);
  
    private static final String OPTION_INPUT_DIRECTORY = "i";
    private static final String OPTION_DB_HOST = "H";
    private static final String OPTION_DB_PORT = "p";
    private static final String OPTION_DB_NAME = "d";
  
    private static final String DEFAULT_DB_HOST = "localhost";
    private static final String DEFAULT_DB_PORT = "27017";
    private static final String DEFAULT_DB_NAME = "actv01";
  
    public static final String HELP_MESSAGE = StringUtils.join(new String[]{
        "This class parses HMDB XML files, converts them into chemical documents, and stores them in a DB",
    }, "");
  
    public static final List<Option.Builder> OPTION_BUILDERS = new ArrayList<Option.Builder>() {{
      add(Option.builder(OPTION_INPUT_DIRECTORY)
          .argName("input dir")
          .desc("The directory where the HMDB XML files live")
          .hasArg()
          .required()
          .longOpt("input-dir")
      );
      add(Option.builder(OPTION_DB_HOST)
          .argName("hostname")
          .desc(String.format("The DB host to which to connect (default: %s)", DEFAULT_DB_HOST))
          .hasArg()
          .longOpt("db-host")
      );
      add(Option.builder(OPTION_DB_PORT)
          .argName("port")
          .desc(String.format("The DB port to which to connect (default: %s)", DEFAULT_DB_PORT))
          .hasArg()
          .longOpt("db-port")
      );
      add(Option.builder(OPTION_DB_NAME)
          .argName("name")
          .desc(String.format("The name of the DB to which to install the HMDB chemicals (default: %s)", DEFAULT_DB_NAME))
          .hasArg()
          .longOpt("db-name")
      );
      add(Option.builder("h")
          .argName("help")
          .desc("Prints this help message")
          .longOpt("help")
      );
    }};
  
    public static final HelpFormatter HELP_FORMATTER = new HelpFormatter();
    static {
      HELP_FORMATTER.setWidth(100);
    }
  
    /* HMDB files all have five digits from 1 through 61388 as of the initial writing of this class.  I've allowed for an
     * extra digit in case the next release of the DB exceeds 100k metabolites.  We also log rejected files just in case.
     */
    private static final Pattern HMDB_FILE_REGEX = Pattern.compile("^HMDB\\d{5,6}\\.xml$");
  
    /* Represent the HMDB paths as enums to constrain the universe of extracted features to a fixed set of paths.
     * Most of the features are not sub-tree dependent, so we can just separate them into TEXT and NODES paths (i.e. paths
     * that return a single string or paths that return nodes containing either a string or a sub-tree that needs to be
     * re-parsed together).  For sub-trees requiring dependent parsing, where one path doesn't do the trick, we use the
     * L1/L2 convention adopted in PubchemParser:
     *   Structure: <feature name>_<level>[_<sub-feature or structure>]_<type>
     * L1 extracts a sub-tree, while L2 extracts the features of the sub-tree so they can be linked together.
     *
     * Note also the description of features we're *not* currently extracting.  There are other data in the HMDB entries
     * that may be useful at some point, but in the interest of time are being ignored for now. */
    private enum HMDB_XPATH {
      HMDB_ID_TEXT("/metabolite/accession/text()"),
      // Names
      PRIMARY_NAME_TEXT("/metabolite/name/text()"),
      IUPAC_NAME_TEXT("/metabolite/iupac_name/text()"),
      SYNONYMS_NODES("/metabolite/synonyms/synonym"),
      // Structures
      INCHI_TEXT("/metabolite/inchi/text()"),
      SMILES_TEXT("/metabolite/smiles/text()"),
      // Ontology
      ONTOLOGY_STATUS_TEXT("/metabolite/ontology/status/text()"),
      ONTOLOGY_ORIGINS_NODES("/metabolite/ontology/origins/origin"),
      ONTOLOGY_FUNCTIONS_NODES("/metabolite/ontology/functions/function"),
      ONTOLOGY_APPLICATIONS_NODES("/metabolite/ontology/applications/application"),
      ONTOLOGY_LOCATIONS_NODES("/metabolite/ontology/cellular_locations/cellular_location"),
      // Physiological locality
      LOCATIONS_FLUID_NODES("/metabolite/biofluid_locations/biofluid"),
      LOCATIONS_TISSUE_NODES("/metabolite/tissue_locations/tissue"),
      // Metabolic pathways
      PATHWAY_NAME_NODES("/metabolite/pathways/pathway/name"),
      // Diseases
      DISEASE_NAME_NODES("/metabolite/diseases/disease/name"),
      // External IDs
      METLIN_ID_TEXT("/metabolite/metlin_id/text()"),
      PUBCHEM_ID_TEXT("/metabolite/pubchem_compound_id/text()"),
      CHEBI_ID_TEXT("/metabolite/chebi_id/text()"),
      // Proteins
      PROTEIN_L1_NODES("/metabolite/protein_associations/protein"),
      PROTEIN_L2_NAME_TEXT("/protein/name/text()"),
      PROTEIN_L2_UNIPROT_ID_TEXT("/protein/uniprot_id/text()"),
      PROTEIN_L2_GENE_NAME_TEXT("/protein/gene_name/text()"),
  
      /* Features we're not extracting right now:
       * * Normal/abnormal concentrations in different fluids/tissues (too many different kinds of expression/units)
       * * Experimentally derived and predicted properties (many of the latter come from Chemaxon anyway)
       * * "specdb" ids, which represent NRM/MS2 data out there, not sure how useful this is right now
       * * Pathway details and ids, which hopefully are already captured via Metacyc
       * * Literature references, which we'd only inspect manually at present and we can always return to the source
       */
      ;
  
      String path;
  
      HMDB_XPATH(String path) {
        this.path = path;
      }
  
      public String getPath() {
        return path;
      }
  
      // We rely on Jaxen because we've experienced performance problems using the built-in/xerces XPath implementation.
      DOMXPath compile() throws JaxenException {
        return new DOMXPath(this.getPath());
      }
    }
  
    public static void main(String[] args) throws Exception {
      // Parse the command line options
      Options opts = new Options();
      for (Option.Builder b : OPTION_BUILDERS) {
        opts.addOption(b.build());
      }
  
      CommandLine cl = null;
      try {
        CommandLineParser parser = new DefaultParser();
        cl = parser.parse(opts, args);
      } catch (ParseException e) {
        System.err.format("Argument parsing failed: %s\n", e.getMessage());
        HELP_FORMATTER.printHelp(PubchemParser.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        System.exit(1);
      }
  
      if (cl.hasOption("help")) {
        HELP_FORMATTER.printHelp(PubchemParser.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        return;
      }
  
      File inputDir = new File(cl.getOptionValue(OPTION_INPUT_DIRECTORY));
      if (!inputDir.isDirectory()) {
        System.err.format("Input directory at %s is not a directory\n", inputDir.getAbsolutePath());
        System.exit(1);
      }
  
      String dbName = cl.getOptionValue(OPTION_DB_NAME, DEFAULT_DB_NAME);
      String dbHost = cl.getOptionValue(OPTION_DB_HOST, DEFAULT_DB_HOST);
      Integer dbPort = Integer.valueOf(cl.getOptionValue(OPTION_DB_PORT, DEFAULT_DB_PORT));
  
      LOGGER.info("Connecting to %s:%d/%s", dbHost, dbPort, dbName);
      MongoDB db = new MongoDB(dbHost, dbPort, dbName);
      HMDBParser parser = Factory.makeParser(db);
  
      LOGGER.info("Starting parser");
      parser.run(inputDir);
      LOGGER.info("Done");
    }
  
    private final Map<HMDB_XPATH, XPath> xpaths = new HashMap<>();
    private MongoDB db;
    // This is required for extracting locality-dependent features of sub-trees using XPath.
    private DocumentBuilder documentBuilder;
  
    protected HMDBParser(MongoDB db) {
      this.db = db;
    }
  
    protected void init() throws JaxenException, ParserConfigurationException {
      for (HMDB_XPATH xpath : HMDB_XPATH.values()) {
        xpaths.put(xpath, xpath.compile());
      }
  
      // This bit pilfered from PubchemParser.java.
      // TODO: next time we use this, put it in a common super class (do it once, do it again, then do it right!).
      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
      documentBuilder = factory.newDocumentBuilder();
    }
  
  
    // TODO: add some constraints on HMDB_XPATHs that allow us to programmatically check they're being applied correctly.
    /**
     * Get the text contents contained in a list of nodes.  Used for multi-valued fields that are siblings in the tree.
     * @param n A list of nodes whose text content should be extracted.
     * @return The text for each node.
     */
    private static List<String> extractNodesText(List<Node> n) {
      return n.stream().map(Node::getTextContent).collect(Collectors.toList());
    }
  
    private List<Node> getNodes(HMDB_XPATH xpath, Document doc) throws JaxenException {
      return (List<Node>) xpaths.get(xpath).selectNodes(doc); // No check, but guaranteed to return List<Node>.
    }
  
    /**
     * Extract the textual content for a set of sibling nodes appearing at some path in the specified document.
     * @param xpath The path to use as a query.
     * @param doc The document to query.
     * @return The textual content of the nodes that live at the specified path.
     * @throws JaxenException
     */
    private List<String> getTextFromNodes(HMDB_XPATH xpath, Document doc) throws JaxenException {
      return extractNodesText(getNodes(xpath, doc));
    }
  
    private String getText(HMDB_XPATH xpath, Document doc) throws JaxenException {
      return xpaths.get(xpath).stringValueOf(doc);
    }
  
    /**
     * Convert an HMDB XML document into a Chemical object.  Expects one chemical per document.
     * @param doc A parsed HMDB XML doc.
     * @return The corresponding chemical to store in the DB.
     * @throws JaxenException
     * @throws JSONException
     */
    protected Chemical extractChemicalFromXMLDocument(Document doc) throws JaxenException, JSONException {
      String hmdbId = getText(HMDB_XPATH.HMDB_ID_TEXT, doc);
  
      String primaryName = getText(HMDB_XPATH.PRIMARY_NAME_TEXT, doc);
      String iupacName = getText(HMDB_XPATH.IUPAC_NAME_TEXT, doc);
      List<String> synonyms = getTextFromNodes(HMDB_XPATH.SYNONYMS_NODES, doc);
  
      String inchi = getText(HMDB_XPATH.INCHI_TEXT, doc);
      String smiles = getText(HMDB_XPATH.SMILES_TEXT, doc);
  
      // Require an InChI if we're going to consume this molecule.
      if (inchi == null || inchi.isEmpty()) {
        LOGGER.warn("No InChI found for HMDB chemical %s, aborting", hmdbId);
        return null;
      }
  
      String ontologyStatus = getText(HMDB_XPATH.ONTOLOGY_STATUS_TEXT, doc);
      List<String> ontologyOrigins = getTextFromNodes(HMDB_XPATH.ONTOLOGY_ORIGINS_NODES, doc);
      List<String> ontologyFunctions = getTextFromNodes(HMDB_XPATH.ONTOLOGY_FUNCTIONS_NODES, doc);
      List<String> ontologyApplications = getTextFromNodes(HMDB_XPATH.ONTOLOGY_APPLICATIONS_NODES, doc);
      List<String> ontologyLocations = getTextFromNodes(HMDB_XPATH.ONTOLOGY_LOCATIONS_NODES, doc);
  
      List<String> locationFluids = getTextFromNodes(HMDB_XPATH.LOCATIONS_FLUID_NODES, doc);
      List<String> locationTissues = getTextFromNodes(HMDB_XPATH.LOCATIONS_TISSUE_NODES, doc);
  
      List<String> pathwayNames = getTextFromNodes(HMDB_XPATH.PATHWAY_NAME_NODES, doc);
  
      List<String> diseaseNames = getTextFromNodes(HMDB_XPATH.DISEASE_NAME_NODES, doc);
  
      String metlinId = getText(HMDB_XPATH.METLIN_ID_TEXT, doc);
      String pubchemId = getText(HMDB_XPATH.PUBCHEM_ID_TEXT, doc);
      String chebiId = getText(HMDB_XPATH.CHEBI_ID_TEXT, doc);
  
      List<Node> proteins = getNodes(HMDB_XPATH.PROTEIN_L1_NODES, doc);
      // Simple triples of name, uniprot id, gene name.
      List<Triple<String, String, String>> proteinAttributes = new ArrayList<>(proteins.size());
  
      for (Node n : proteins) {
        /* In order to run XPath on a sub-document, we have to Extract the relevant nodes into their own document object.
         * If we try to run evaluate on `n` instead of this new document, we'll get matching paths for the original
         * document `d` but not for the nodes we're looking at right now.  Very weird. */
        Document proteinDoc = documentBuilder.newDocument();
        proteinDoc.adoptNode(n);
        proteinDoc.appendChild(n);
        String name = getText(HMDB_XPATH.PROTEIN_L2_NAME_TEXT, proteinDoc);
        String uniprotId = getText(HMDB_XPATH.PROTEIN_L2_UNIPROT_ID_TEXT, proteinDoc);
        String geneName = getText(HMDB_XPATH.PROTEIN_L2_GENE_NAME_TEXT, proteinDoc);
        proteinAttributes.add(Triple.of(name, uniprotId, geneName));
      }
  
      // Assumption: when we reach this point there will always be an InChI.
      Chemical chem = new Chemical(inchi);
      chem.setSmiles(smiles);
  
      chem.setCanon(primaryName);
  
      if (pubchemId != null && !pubchemId.isEmpty()) {
        chem.setPubchem(Long.valueOf(pubchemId));
      }
  
      synonyms.forEach(chem::addSynonym);
      chem.addSynonym(iupacName); // TODO: is there a better place for this?
  
      JSONObject meta = new JSONObject()
          .put("hmdb_id", hmdbId)
          .put("ontology", new JSONObject()
              .put("status", ontologyStatus)
              .put("origins", new JSONArray(ontologyOrigins))
              .put("functions", new JSONArray(ontologyFunctions))
              .put("applications", new JSONArray(ontologyApplications))
              .put("locations", new JSONArray(ontologyLocations))
          )
          .put("location", new JSONObject()
              .put("fluid", new JSONArray(locationFluids))
              .put("tissue", new JSONArray(locationTissues))
          )
          .put("pathway_names", new JSONArray(pathwayNames))
          .put("disease_names", new JSONArray(diseaseNames))
          .put("metlin_id", metlinId)
          .put("chebi_id", chebiId)
          .put("proteins", new JSONArray(proteinAttributes.stream()
              .map(t -> new JSONObject().
                  put("name", t.getLeft()).
                  put("uniprot_id", t.getMiddle()).
                  put("gene_name", t.getRight())
              ).collect(Collectors.toList())
          )
      );
  
      chem.putRef(Chemical.REFS.HMDB, meta);
  
      return chem;
    }
  
    protected SortedSet<File> findHMDBFilesInDirectory(File dir) throws IOException {
      // Sort for consistency + sanity.
      SortedSet<File> results = new TreeSet<>((a, b) -> a.getName().compareTo(b.getName()));
      for (File file : dir.listFiles()) { // Do our own filtering so we can log rejects, of which we expect very few.
        if (HMDB_FILE_REGEX.matcher(file.getName()).matches()) {
          results.add(file);
        } else {
          LOGGER.warn("Found non-conforming HMDB file in directory %s: %s", dir.getAbsolutePath(), file.getName());
        }
      }
      return results;
    }
  
    /**
     * Extract all chemicals from HMDB XML files that live in the specified directory and save them in the DB.
     * Note that this search is not recursive: documents in sub-directories will be ignored.
     * @param inputDir The directory to scan for HMDB XML files.
     * @throws IOException
     * @throws IllegalArgumentException
     */
    public void run(File inputDir) throws IOException, IllegalArgumentException {
      if (inputDir == null || !inputDir.isDirectory()) {
        String msg = String.format("Unable to read input directory at %s",
            inputDir == null ? "null" : inputDir.getAbsolutePath());
        LOGGER.error(msg);
        throw new RuntimeException(msg);
      }
  
      SortedSet<File> files = findHMDBFilesInDirectory(inputDir);
      LOGGER.info("Found %d HMDB XML files in directory %s", files.size(), inputDir.getAbsolutePath());
  
      for (File file : files) {
        LOGGER.debug("Processing HMDB XML file %s", file.getAbsolutePath());
  
        /* Promote our XML-specific exceptions to generic IllegalArgumentExceptions to reduce error handling surface
         * area for the caller. */
        Document d;
        try {
          d = documentBuilder.parse(file);
        } catch (SAXException e) {
          String msg = String.format("Unable to parse XML file at %s: %s", file.getAbsolutePath(), e.getMessage());
          throw new IllegalArgumentException(msg, e);
        }
  
        /* Jaxen doesn't throw exceptions if it can't find a path, so a JaxenException here is completely unexpected.
         * It might mean corrupted XML or some unrecoverable XPath problem that we don't expect.  In any case, promote
         * the exception to the caller as it's unclear how we could deal with such an error here. */
        Chemical chem;
        try {
          chem = extractChemicalFromXMLDocument(d);
        } catch (JaxenException e) {
          String msg = String.format("Unable to extract features from XML file at %s: %s",
              file.getAbsolutePath(), e.getMessage());
          throw new IllegalArgumentException(msg, e);
        }
  
        // Not all HMDB entries contain
        if (chem == null) {
          LOGGER.warn("Unable to create chemical from file %s", file.getAbsolutePath());
          continue;
        }
  
        // submitToActChemicalDB creates or merges as necessary.
        Long id = db.getNextAvailableChemicalDBid();
        db.submitToActChemicalDB(chem, id);
        LOGGER.debug("Submitted chemical %d to the DB", id);
      }
      LOGGER.info("Loaded %d HMDB chemicals into DB", files.size());
    }
  
    public static class Factory {
      public static HMDBParser makeParser(MongoDB db) {
        HMDBParser parser = new HMDBParser(db);
        // Promote XML-specific exceptions from parser initialization to runtime exceptions, as they are definite bugs.
        try {
          parser.init();
        } catch (JaxenException e) {
          LOGGER.error("BUG: caught JaxenException on initialization, which means programmer error: %s",
              e.getMessage());
          throw new RuntimeException(e);
        } catch (ParserConfigurationException e) {
          LOGGER.error("BUG: caught ParserConfigurationException on initialization, which means programmer error: %s",
              e.getMessage());
          throw new RuntimeException(e);
        }
  
        return parser;
      }
    }
  }
  