content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.twentyn.patentExtractor;
  
  import com.fasterxml.jackson.annotation.JsonProperty;
  import nu.xom.converters.DOMConverter;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  import org.w3c.dom.DOMImplementation;
  import org.w3c.dom.Document;
  import org.w3c.dom.Element;
  import org.w3c.dom.Node;
  import org.w3c.dom.NodeList;
  import uk.ac.cam.ch.wwmm.chemicaltagger.ChemistryPOSTagger;
  import uk.ac.cam.ch.wwmm.chemicaltagger.ChemistrySentenceParser;
  import uk.ac.cam.ch.wwmm.chemicaltagger.POSContainer;
  
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.ParserConfigurationException;
  import javax.xml.transform.Transformer;
  import javax.xml.transform.TransformerException;
  import javax.xml.transform.TransformerFactory;
  import javax.xml.transform.dom.DOMSource;
  import javax.xml.transform.stream.StreamResult;
  import javax.xml.xpath.XPath;
  import javax.xml.xpath.XPathConstants;
  import javax.xml.xpath.XPathExpressionException;
  import java.io.IOException;
  import java.io.StringWriter;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  
  public class PatentDocumentFeatures {
    public static final Logger LOGGER = LogManager.getLogger(PatentDocumentFeatures.class);
  
    public static final String SENTENCE_PATH = "//MOLECULE//ancestor::Sentence";
    public static final String SENTENCE_DOC_HEADER = "molecule-sentence";
    public static final String MOLECULE_PATH = "//MOLECULE";
  
    // TODO: nullable/nonnull annotations
    private static List<Document> runTagger(DocumentBuilder docBuilder, ChemistryPOSTagger tagger,
                                            List<String> textContent)
        throws ParserConfigurationException, XPathExpressionException {
      List<Document> tagDocs = new ArrayList<>(textContent.size());
      for (String text : textContent) {
        POSContainer container = tagger.runTaggers(text);
        ChemistrySentenceParser parser = new ChemistrySentenceParser(container);
        parser.parseTags();
        nu.xom.Document xomDoc = parser.makeXMLDocument();
        DOMImplementation domImpl = docBuilder.getDOMImplementation();
        Document doc = DOMConverter.convert(xomDoc, domImpl);
        tagDocs.add(doc);
      }
      return tagDocs;
    }
  
    /**
     * Extracts sentence nodes from a POS-tagger XML document.  These sentences are intended to provide some notion of
     * locality for identified chemical entities.
     *
     * @param docBuilder A document builder to use when producing single-sentence XML documents.
     * @param doc        The POS-tagger XML document from which to extract sentences.
     * @return A list of single-sentence documents.
     * @throws ParserConfigurationException
     * @throws XPathExpressionException
     */
    private static List<Document> findSentences(DocumentBuilder docBuilder, Document doc)
        throws ParserConfigurationException, XPathExpressionException {
      if (doc != null) {
        // TODO: is there a more efficient yet still safe way to do this?
        XPath xpath = Util.getXPathFactory().newXPath();
        // TODO: get rid of this inline xpath compilation, run during setup.
        NodeList nodes =
            (NodeList) xpath.evaluate(SENTENCE_PATH, doc, XPathConstants.NODESET);
  
        List<Document> docList = new ArrayList<>(nodes.getLength());
        for (int i = 0; i < nodes.getLength(); i++) {
          Node n = nodes.item(i);
  
          /* With help from:
           * http://examples.javacodegeeks.com/core-java/xml/dom/copy-nodes-subtree-from-one-dom-document-to-another/ */
          org.w3c.dom.Document newDoc = docBuilder.newDocument();
          Element rootElement = newDoc.createElement(SENTENCE_DOC_HEADER);
          Node newNode = newDoc.importNode(n, true);
          rootElement.appendChild(newNode);
          newDoc.appendChild(rootElement);
          docList.add(newDoc);
        }
        return docList;
      } else {
        // TODO: log here.
        return new ArrayList<>(0);
      }
    }
  
    /* Node.getTextContent() returns the concatenation of all text content without any delimitation between nodes.
     * This function recursively traverses the document structure, appending (naively) text content to a string joiner
     * as it goes. */
    private static List<String> appendTextContent(List<String> textList, Node n) {
      if (n.getNodeType() == Node.TEXT_NODE) {
        textList.add(n.getTextContent());
      } else {
        NodeList childNodes = n.getChildNodes();
        for (int j = 0; j < childNodes.getLength(); j++) {
          Node childNode = childNodes.item(j);
          textList = appendTextContent(textList, childNode);
        }
      }
      return textList;
    }
  
    private static Map<String, Integer> extractMoleculeCounts(Map<String, Integer> moleculeCounts, Document doc)
        throws ParserConfigurationException, XPathExpressionException {
      if (doc != null) {
        /* This uses //MOLECULE instead of //MOLECULE//text(), as the latter finds all text for all molecules
         * instead of text for each molecule.  We could also do a secondary traversal of each MOLECULE fragment,
         * but running XPath queries over XPath results is a major pain.  Instead, we'll grab the MOLECULE nodes
         * and recursively extract the text content one molecule at a time. */
        XPath xpath = Util.getXPathFactory().newXPath();
        NodeList nodes =
            (NodeList) xpath.evaluate(MOLECULE_PATH, doc, XPathConstants.NODESET);
        for (int i = 0; i < nodes.getLength(); i++) {
          List<String> nameList = appendTextContent(new LinkedList<String>(), nodes.item(i));
          String moleculeName = StringUtils.join(nameList, " ");
          Integer count = moleculeCounts.get(moleculeName);
          if (count == null) {
            count = 0;
          }
          moleculeCounts.put(moleculeName, count + 1);
        }
      }
      return moleculeCounts;
    }
  
    private static String docToString(Transformer transformer, Document doc) throws TransformerException {
      StringWriter stringWriter = new StringWriter();
      DOMSource source = new DOMSource(doc);
      StreamResult result = new StreamResult(stringWriter);
      transformer.transform(source, result);
      return stringWriter.toString();
    }
  
    // TODO: prolly belongs in a factory.
  
    /**
     * Extracts features from PatentDocument objects, including counts of terms in the patent text that can be
     * identified as chemical entities.
     *
     * @param posTagger      A ChemTagger POS (part of speech) tagger to use when extracting features from the patent text.
     * @param patentDocument The PatentDocument from which to extract features.
     * @return A PatentDocumentFeatures object containing features for the specified patent document.
     * @throws ParserConfigurationException
     * @throws XPathExpressionException
     * @throws TransformerException
     * @throws IOException
     */
    public static PatentDocumentFeatures extractPatentDocumentFeatures(
        ChemistryPOSTagger posTagger, PatentDocument patentDocument)
        throws ParserConfigurationException, XPathExpressionException, TransformerException, IOException {
      DocumentBuilder docBuilder = Util.mkDocBuilderFactory().newDocumentBuilder();
  
      List<Document> claimsDocs = runTagger(docBuilder, posTagger, patentDocument.getClaimsText());
      List<Document> textDocs = runTagger(docBuilder, posTagger, patentDocument.getTextContent());
  
      List<Document> claimsTags = new ArrayList<>(claimsDocs.size());
      for (Document d : claimsDocs) {
        List<Document> sentences = findSentences(docBuilder, d);
        claimsTags.addAll(sentences);
      }
      List<Document> textTags = new ArrayList<>(textDocs.size());
      for (Document d : textDocs) {
        List<Document> sentences = findSentences(docBuilder, d);
        textTags.addAll(sentences);
      }
  
      Transformer transformer = TransformerFactory.newInstance().newTransformer();
  
      List<String> claimsSentences = new ArrayList<>(claimsTags.size());
      for (Document doc : claimsTags) {
        claimsSentences.add(docToString(transformer, doc));
      }
  
      List<String> textSentences = new ArrayList<>(textTags.size());
      for (Document doc : textTags) {
        textSentences.add(docToString(transformer, doc));
      }
  
      Map<String, Integer> claimsMoleculeCounts = new HashMap<>();
      for (Document doc : claimsDocs) {
        extractMoleculeCounts(claimsMoleculeCounts, doc);
      }
      Map<String, Integer> textMoleculeCounts = new HashMap<>();
      for (Document doc : textDocs) {
        extractMoleculeCounts(textMoleculeCounts, doc);
      }
  
      return new PatentDocumentFeatures(patentDocument,
          claimsDocs, textDocs,
          claimsSentences, textSentences,
          claimsMoleculeCounts, textMoleculeCounts);
    }
  
  
    @JsonProperty("patent_document")
    protected PatentDocument patentDocument;
    // TODO: why are JsonSerialize and JsonDeserialize ignored in this situation (hence they're commented out.)?
    // @JsonSerialize(contentUsing = Util.DocumentSerializer.class)
    // @JsonDeserialize(contentUsing = Util.DocumentDeserializer.class)
    @JsonProperty("claims_tags")
    protected List<Document> claimsDocuments;
    // @JsonSerialize(contentUsing = Util.DocumentSerializer.class)
    // @JsonDeserialize(contentUsing = Util.DocumentDeserializer.class)
    @JsonProperty("text_tags")
    protected List<Document> textDocuments;
    @JsonProperty("claims_sentences")
    protected List<String> claimsSentences;
    @JsonProperty("text_sentences")
    protected List<String> textSentences;
    @JsonProperty("claims_molecule_counts")
    protected Map<String, Integer> claimsMoleculeCounts;
    @JsonProperty("text_molecule_counts")
    protected Map<String, Integer> textMoleculeCounts;
  
    public PatentDocumentFeatures(PatentDocument patentDocument,
                                  List<Document> claimsDocuments, List<Document> textDocuments,
                                  List<String> claimsSentences, List<String> textSentences,
                                  Map<String, Integer> claimsMoleculeCounts, Map<String, Integer> textMoleculeCounts) {
      this.patentDocument = patentDocument;
      this.claimsDocuments = claimsDocuments;
      this.textDocuments = textDocuments;
      this.claimsSentences = claimsSentences;
      this.textSentences = textSentences;
      this.claimsMoleculeCounts = claimsMoleculeCounts;
      this.textMoleculeCounts = textMoleculeCounts;
    }
  
    public PatentDocument getPatentDocument() {
      return patentDocument;
    }
  
    public List<Document> getClaimsDocument() {
      return claimsDocuments;
    }
  
    public List<Document> getTextDocument() {
      return textDocuments;
    }
  
    public List<String> getClaimsSentences() {
      return claimsSentences;
    }
  
    public List<String> getTextSentences() {
      return textSentences;
    }
  
    public Map<String, Integer> getClaimsMoleculeCounts() {
      return claimsMoleculeCounts;
    }
  
    public Map<String, Integer> getTextMoleculeCounts() {
      return textMoleculeCounts;
    }
  }
  