content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.twentyn.patentTextProcessor;
  
  import com.fasterxml.jackson.annotation.JsonProperty;
  import com.fasterxml.jackson.databind.ObjectMapper;
  import com.fasterxml.jackson.databind.SerializationFeature;
  import com.twentyn.patentExtractor.PatentCorpusReader;
  import com.twentyn.patentExtractor.PatentDocument;
  import com.twentyn.patentExtractor.PatentProcessor;
  import edu.emory.clir.clearnlp.component.utils.NLPUtils;
  import edu.emory.clir.clearnlp.tokenization.AbstractTokenizer;
  import edu.emory.clir.clearnlp.util.lang.TLanguage;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.DefaultParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.commons.io.input.ReaderInputStream;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.log4j.Logger;
  import org.xml.sax.SAXException;
  
  import javax.xml.parsers.ParserConfigurationException;
  import javax.xml.transform.TransformerConfigurationException;
  import javax.xml.transform.TransformerException;
  import javax.xml.xpath.XPathExpressionException;
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileReader;
  import java.io.IOException;
  import java.io.Reader;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.regex.Pattern;
  
  public class WordCountProcessor implements PatentProcessor {
  
    public static final Logger LOGGER = Logger.getLogger(PatentProcessor.class);
  
    public static class PatentDocumentWordCount {
      @JsonProperty("patent_file_path")
      String patentFilePath;
      @JsonProperty("word_count")
      Map<String, Integer> wordCount;
  
      public PatentDocumentWordCount(String patentFilePath, Map<String, Integer> wordCount) {
        this.patentFilePath = patentFilePath;
        this.wordCount = wordCount;
      }
  
      public String getPatentFilePath() {
        return patentFilePath;
      }
  
      public Map<String, Integer> getWordCount() {
        return wordCount;
      }
    }
  
    ObjectMapper objectMapper = new ObjectMapper();
    {
      objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
      objectMapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
    }
  
    private AbstractTokenizer tokenizer;
    private Set<String> stopWords = new HashSet<>();
    public WordCountProcessor() {
      this.tokenizer = NLPUtils.getTokenizer(TLanguage.ENGLISH);
    }
  
    public void readStopWords(File path) throws IOException{
      try (BufferedReader r = new BufferedReader(new FileReader(path))) {
        String line = null;
        while ((line = r.readLine()) != null) {
          this.stopWords.add(line);
        }
      }
    }
  
    public static final Pattern EXCLUDE_NUMBERS = Pattern.compile("^[+\\-]?[0-9\\.]+$");
  
    public void processPatentText(File patentFile, Reader patentTextReader, int patentTextLength)
        throws IOException, ParserConfigurationException, SAXException,
        TransformerConfigurationException, TransformerException, XPathExpressionException {
  
      PatentDocument doc = PatentDocument.patentDocumentFromXMLStream(new ReaderInputStream(patentTextReader));
      if (doc == null) {
        LOGGER.warn(String.format("Got null patent document object for patent at %s", patentFile.getAbsolutePath()));
        return;
      }
      // Put all the text back together into one big string.  Sigh.
      List<String> textFields = new ArrayList<String>(1 + doc.getClaimsText().size() + doc.getTextContent().size());
      textFields.add(doc.getTitle());
      textFields.addAll(doc.getClaimsText());
      textFields.addAll(doc.getTextContent());
      String joinedText = StringUtils.join(textFields, "\n");
  
      // TODO: parallelize this.  Because come on, it's the nineties and Map Reduce is almost a thousand years old.
      Map<String, Integer> wordCount = new HashMap<>();
      List<String> words = this.tokenizer.tokenize(joinedText);
      for (String word : words) {
        if (EXCLUDE_NUMBERS.matcher(word).matches()) {
          continue; // Ignore words that are just numbers (with decimals)
        }
  
        word = word.toLowerCase();
        if (stopWords.contains(word)) {
          continue; // Ignore stop words;
        }
  
        Integer count = wordCount.get(word);
        if (count == null) {
          count = 1;
        }
        wordCount.put(word, count + 1);
      }
  
      PatentDocumentWordCount pdwc = new PatentDocumentWordCount(patentFile.getAbsolutePath(), wordCount);
      System.out.println(objectMapper.writeValueAsString(pdwc));
    }
  
    public static void main(String[] args) throws Exception {
      System.out.println("Starting up...");
      System.out.flush();
      Options opts = new Options();
      opts.addOption(Option.builder("i").
          longOpt("input").hasArg().required().desc("Input file or directory to score").build());
      opts.addOption(Option.builder("h").longOpt("help").desc("Print this help message and exit").build());
      opts.addOption(Option.builder("v").longOpt("verbose").desc("Print verbose log output").build());
  
  
      HelpFormatter helpFormatter = new HelpFormatter();
      CommandLineParser cmdLineParser = new DefaultParser();
      CommandLine cmdLine = null;
      try {
        cmdLine = cmdLineParser.parse(opts, args);
      } catch (ParseException e) {
        System.out.println("Caught exception when parsing command line: " + e.getMessage());
        helpFormatter.printHelp("WordCountProcessor", opts);
        System.exit(1);
      }
  
      if (cmdLine.hasOption("help")) {
        helpFormatter.printHelp("DocumentIndexer", opts);
        System.exit(0);
      }
  
      String inputFileOrDir = cmdLine.getOptionValue("input");
      File splitFileOrDir = new File(inputFileOrDir);
      if (!(splitFileOrDir.exists())) {
        LOGGER.error("Unable to find directory at " + inputFileOrDir);
        System.exit(1);
      }
  
      WordCountProcessor wcp = new WordCountProcessor();
      PatentCorpusReader corpusReader = new PatentCorpusReader(wcp, splitFileOrDir);
      corpusReader.readPatentCorpus();
    }
  }
  