content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.twentyn.patentSearch;
  
  import com.twentyn.patentExtractor.PatentCorpusReader;
  import com.twentyn.patentExtractor.PatentDocument;
  import com.twentyn.patentExtractor.PatentProcessor;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.DefaultParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.commons.io.input.ReaderInputStream;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.logging.log4j.Level;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  import org.apache.logging.log4j.core.LoggerContext;
  import org.apache.logging.log4j.core.config.Configuration;
  import org.apache.logging.log4j.core.config.LoggerConfig;
  import org.apache.lucene.analysis.Analyzer;
  import org.apache.lucene.analysis.custom.CustomAnalyzer;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.document.Field;
  import org.apache.lucene.document.SortedSetDocValuesField;
  import org.apache.lucene.document.StringField;
  import org.apache.lucene.document.TextField;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.index.IndexWriterConfig;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.FSDirectory;
  import org.apache.lucene.util.BytesRef;
  import org.xml.sax.SAXException;
  
  import javax.xml.parsers.ParserConfigurationException;
  import javax.xml.transform.TransformerConfigurationException;
  import javax.xml.transform.TransformerException;
  import javax.xml.xpath.XPathExpressionException;
  import java.io.File;
  import java.io.IOException;
  import java.io.Reader;
  import java.nio.charset.Charset;
  
  /**
   * This class reads concatenated USPTO XML patent documents (as distributed in Google's patent full-text corpus) and
   * indexes their contents with minimal normalization.  Lucene is used for indexing; the output of this class's main
   * method is a Lucene index of the contents of any specified patent files.  A single file or directory of files can be
   * supplied as input.  If a directory is specified, this class will read and index any .zip files in that directory,
   * assuming these to be compressed USPTO dumps.
   */
  public class DocumentIndexer implements PatentProcessor {
    public static final Logger LOGGER = LogManager.getLogger(DocumentIndexer.class);
  
    public static void main(String[] args) throws Exception {
      System.out.println("Starting up...");
      System.out.flush();
      Options opts = new Options();
      opts.addOption(Option.builder("i").
          longOpt("input").hasArg().required().desc("Input file or directory to index").build());
      opts.addOption(Option.builder("x").
          longOpt("index").hasArg().required().desc("Path to index file to generate").build());
      opts.addOption(Option.builder("h").longOpt("help").desc("Print this help message and exit").build());
      opts.addOption(Option.builder("v").longOpt("verbose").desc("Print verbose log output").build());
  
      HelpFormatter helpFormatter = new HelpFormatter();
      CommandLineParser cmdLineParser = new DefaultParser();
      CommandLine cmdLine = null;
      try {
        cmdLine = cmdLineParser.parse(opts, args);
      } catch (ParseException e) {
        System.out.println("Caught exception when parsing command line: " + e.getMessage());
        helpFormatter.printHelp("DocumentIndexer", opts);
        System.exit(1);
      }
  
      if (cmdLine.hasOption("help")) {
        helpFormatter.printHelp("DocumentIndexer", opts);
        System.exit(0);
      }
  
      if (cmdLine.hasOption("verbose")) {
        // With help from http://stackoverflow.com/questions/23434252/programmatically-change-log-level-in-log4j2
        LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
        Configuration ctxConfig = ctx.getConfiguration();
        LoggerConfig logConfig = ctxConfig.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
        logConfig.setLevel(Level.DEBUG);
  
        ctx.updateLoggers();
        LOGGER.debug("Verbose logging enabled");
      }
  
      LOGGER.info("Opening index at " + cmdLine.getOptionValue("index"));
      Directory indexDir = FSDirectory.open(new File(cmdLine.getOptionValue("index")).toPath());
  
      /* The standard analyzer is too aggressive with chemical entities (it strips structural annotations, for one
       * thing), and the whitespace analyzer doesn't do any case normalization or stop word elimination.  This custom
       * analyzer appears to treat chemical entities better than the standard analyzer without admitting too much
       * cruft to the index. */
      Analyzer analyzer = CustomAnalyzer.builder().
          withTokenizer("whitespace").
          addTokenFilter("lowercase").
          addTokenFilter("stop").
          build();
  
      IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer);
      writerConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
      writerConfig.setRAMBufferSizeMB(1 << 10);
      IndexWriter indexWriter = new IndexWriter(indexDir, writerConfig);
  
      String inputFileOrDir = cmdLine.getOptionValue("input");
      File splitFileOrDir = new File(inputFileOrDir);
      if (!(splitFileOrDir.exists())) {
        LOGGER.error("Unable to find directory at " + inputFileOrDir);
        System.exit(1);
      }
  
      DocumentIndexer indexer = new DocumentIndexer(indexWriter);
      PatentCorpusReader corpusReader = new PatentCorpusReader(indexer, splitFileOrDir);
      corpusReader.readPatentCorpus();
      indexer.commitAndClose();
    }
  
    IndexWriter indexWriter;
  
    public DocumentIndexer(IndexWriter indexWriter) {
      this.indexWriter = indexWriter;
    }
  
    @Override
    public void processPatentText(File patentFile, Reader patentTextReader, int patentTextLength)
        throws IOException, ParserConfigurationException,
        SAXException, TransformerConfigurationException,
        TransformerException, XPathExpressionException {
      PatentDocument patentDocument = PatentDocument.patentDocumentFromXMLStream(
          new ReaderInputStream(patentTextReader, Charset.forName("utf-8")));
      if (patentDocument == null) {
        LOGGER.info("Found non-patent type document, skipping.");
        return;
      }
      Document doc = patentDocToLuceneDoc(patentFile, patentDocument);
      LOGGER.debug("Adding document: " + doc.get("id"));
      this.indexWriter.addDocument(doc);
      this.indexWriter.commit();
    }
  
    public Document patentDocToLuceneDoc(File path, PatentDocument patentDoc) {
      // With help from https://lucene.apache.org/core/5_2_0/demo/src-html/org/apache/lucene/demo/IndexFiles.html
      Document doc = new Document();
      doc.add(new StringField("file_name", path.getName(), Field.Store.YES));
      doc.add(new StringField("id", patentDoc.getFileId(), Field.Store.YES));
      doc.add(new StringField("grant_date", patentDoc.getGrantDate(), Field.Store.YES));
      doc.add(new StringField("main_classification", patentDoc.getMainClassification(), Field.Store.YES));
      doc.add(new TextField("title", patentDoc.getTitle(), Field.Store.YES));
      doc.add(new TextField("claims", StringUtils.join("\n", patentDoc.getClaimsText()), Field.Store.NO));
      doc.add(new TextField("description", StringUtils.join("\n", patentDoc.getTextContent()), Field.Store.NO));
  
      // TODO: verify that these are searchable as expected.
      for (String cls : patentDoc.getFurtherClassifications()) {
        doc.add(new SortedSetDocValuesField("further_classification", new BytesRef(cls)));
      }
      for (String cls : patentDoc.getSearchedClassifications()) {
        doc.add(new SortedSetDocValuesField("searched_classification", new BytesRef(cls)));
      }
  
      return doc;
    }
  
    public void commitAndClose() throws IOException {
      if (!indexWriter.isOpen()) {
        return;
      }
      indexWriter.commit();
      indexWriter.close();
    }
  }
  