content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.twentyn.patentSearch;
  
  import org.apache.commons.lang3.tuple.Pair;
  import org.apache.commons.lang3.tuple.Triple;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.index.DirectoryReader;
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.search.BooleanClause;
  import org.apache.lucene.search.BooleanQuery;
  import org.apache.lucene.search.IndexSearcher;
  import org.apache.lucene.search.PhraseQuery;
  import org.apache.lucene.search.ScoreDoc;
  import org.apache.lucene.search.TermQuery;
  import org.apache.lucene.search.TopDocs;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.FSDirectory;
  
  import java.io.File;
  import java.io.IOException;
  import java.io.UncheckedIOException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.List;
  import java.util.Set;
  import java.util.function.Function;
  import java.util.stream.Collectors;
  import java.util.stream.Stream;
  
  public class Searcher implements AutoCloseable {
    public static final Logger LOGGER = LogManager.getFormatterLogger(Searcher.class);
  
    private static final List<String> KEYWORDS = Collections.unmodifiableList(Arrays.asList(
        "yeast",
        "cerevisiae",
        "coli",
        "biosynthesis",
        "biogenesis",
        "anabolism",
        "catalysis",
        "ferment",
        "fermenter",
        "fermentor",
        "fermentation",
        "fermentive"
    ));
    private static final String CLAIMS_FIELD = "claims";
    private static final int MAX_RESULTS_PER_QUERY = 100;
    // Note: this score is likely dependent on the set of keywords above.  Adjust this if KEYWORDS change.
    private static final float DEFAULT_SCORE_THRESHOLD = 0.1f;
  
    private List<Pair<IndexReader, IndexSearcher>> indexReadersAndSearchers = new ArrayList<>();
    private float scoreThreshold = DEFAULT_SCORE_THRESHOLD;
  
    private Searcher() {
  
    }
  
    private Searcher(float scoreThreshold) {
      this.scoreThreshold = scoreThreshold;
    }
  
    private void init(List<File> indexDirectories) throws IOException {
      for (File indexDirectory : indexDirectories) {
        LOGGER.info("Opening index dir at %s", indexDirectory.getAbsolutePath());
        Directory indexDir = FSDirectory.open(indexDirectory.toPath());
        IndexReader indexReader = DirectoryReader.open(indexDir);
        IndexSearcher searcher = new IndexSearcher(indexReader);
        // Only add to the list if both of these calls work.
        indexReadersAndSearchers.add(Pair.of(indexReader, searcher));
      }
    }
  
    @Override
    public void close() throws IOException {
      for (IndexReader reader : indexReadersAndSearchers.stream().map(Pair::getLeft).collect(Collectors.toList())) {
        try {
          reader.close();
        } catch (IOException e) {
          LOGGER.error("Unable to close index reader, but continuing to try closing others: %s", e.getMessage());
        }
      }
    }
  
    public static class Factory {
      private static final Factory INSTANCE = new Factory();
  
      private Factory() {
  
      }
  
      public static Factory getInstance() {
        return INSTANCE;
      }
  
      public Searcher build(File indexTopDir, float scoreThreshold) throws IOException {
        Searcher s = new Searcher(scoreThreshold);
        runInit(indexTopDir, s);
        return s;
      }
  
      public Searcher build(File indexTopDir) throws IOException {
        Searcher s = new Searcher();
        runInit(indexTopDir, s);
        return s;
      }
  
      private void runInit(File indexTopDir, Searcher s) throws IOException {
        if (!indexTopDir.isDirectory()) {
          String msg = String.format("Top level directory at %s is not a directory", indexTopDir.getAbsolutePath());
          LOGGER.error(msg);
          throw new IOException(msg);
        }
  
        List<File> individualIndexes = Arrays.stream(indexTopDir.listFiles()).
            filter(f -> f.getName().endsWith(".index")).collect(Collectors.toList());
        if (individualIndexes.size() == 0) {
          String msg = String.format("Top level directory at %s contains no index sub-directories",
              indexTopDir.getAbsolutePath());
          LOGGER.error(msg);
          throw new IOException(msg);
        }
  
        s.init(individualIndexes);
      }
    }
  
    /**
     * Search for patents that contain any of the specified chemical synonyms, scored based on synonym and biosynthesis
     * keyword occurrence.  Results are filtered by score.
     * @param synonyms A list of chemical synonyms to use in the search.
     * @return A list of search results whose relevance scores are above the searcher's score threshold.
     * @throws IOException
     */
    public List<SearchResult> searchInClaims(List<String> synonyms) throws IOException {
      if (synonyms.size() == 0) {
        LOGGER.info("Not running search for no synonyms!");
        return Collections.emptyList();
      }
  
      // Make queries for all synonyms.
      final List<BooleanQuery> queries = makeQueries(synonyms, CLAIMS_FIELD).collect(Collectors.toList());
  
      // Reuse the compiled queries for all indices.
      try {
        Set<Triple<Float, String, String>> uniqueResults = indexReadersAndSearchers.stream().
            map(p -> runSearch(p, queries)). // Search to get per-query streams...
            flatMap(Function.identity()).    // combine all the streams into one...
            collect(Collectors.toSet());    // and collect the merged results in a list.
  
        /* Uniq-ify!  It is completely reasonable for a patent to appear for multiple queries.
         * TODO: we haven't seen results appear multiple times with different scores.  We should probably unique-ify
         * on id and take the result with the best score just to be safe. */
        List<Triple<Float, String, String>> results = new ArrayList<>(uniqueResults);
        Collections.sort(results);
  
        return results.stream().
            map(t -> new SearchResult(t.getMiddle(), t.getRight(), t.getLeft())).
            collect(Collectors.toList());
      } catch (UncheckedIOException e) {
        throw e.getCause(); // Promote back to a regular exception for handling by the caller.
      }
    }
  
    // Run a set of queries over a single reader + searcher.
    private Stream<Triple<Float, String, String>> runSearch(
        Pair<IndexReader, IndexSearcher> readerSearcher, List<BooleanQuery> queries) throws UncheckedIOException {
  
      // With hints from http://stackoverflow.com/questions/22382453/java-8-streams-flatmap-method-example
      return queries.stream().map(q -> executeQuery(readerSearcher, q)).flatMap(Collection::stream);
    }
  
    // Run a single query on a single reader + searcher.
    private List<Triple<Float, String, String>> executeQuery(
        Pair<IndexReader, IndexSearcher> readerSearcher, BooleanQuery query) throws UncheckedIOException {
      TopDocs topDocs;
      try {
        topDocs = readerSearcher.getRight().search(query, MAX_RESULTS_PER_QUERY);
      } catch (IOException e) {
        LOGGER.error("Caught IO exception when trying to run search for %s: %s", query, e.getMessage());
        /* Wrap `e` in an unchecked exception to allow it to escape our call stack.  The top level function with catch
         * and rethrow it as a normal IOException. */
        throw new UncheckedIOException(e);
      }
  
      ScoreDoc[] scoreDocs = topDocs.scoreDocs;
      if (scoreDocs.length == 0) {
        LOGGER.debug("Search returned no results.");
        return Collections.emptyList();
      }
      // ScoreDoc just contains a score and an id.  We need to retrieve the documents' content using that id.
  
      /* Crux of the next bit:
       * Filter by score and convert from scoreDocs to document features.
       * No need to use `limit` here since we already had Lucene cap the result set size. */
      return Arrays.stream(scoreDocs).
          filter(scoreDoc -> scoreDoc.score >= scoreThreshold).
          map(scoreDoc -> { //
            try {
              Pair<String, String> features = this.extractDocFeatures(readerSearcher.getLeft().document(scoreDoc.doc));
              // Put the score first so the natural sort order is based on score.
              return Triple.of(scoreDoc.score, features.getLeft(), features.getRight());
            } catch (IOException e) {
              // Yikes, this is v. bad.
              LOGGER.error("Caught IO exception when trying to read doc id %d: %s", scoreDoc.doc, e.getMessage());
              throw new UncheckedIOException(e); // Same as above.
            }
          }).collect(Collectors.toList());
    }
  
    // Just extract the id and title for now.  The id contains the patent number, and the title is enough for display.
    private Pair<String, String> extractDocFeatures(Document doc) {
      return Pair.of(doc.get("id"), doc.get("title"));
    }
  
    private Stream<BooleanQuery> makeQueries(List<String> synonyms, String field) {
      return synonyms.stream().
          filter(syn -> syn != null && !syn.isEmpty()).
          map(syn -> makeQuery(syn, field));
    }
  
    private BooleanQuery makeQuery(String synonym, String field) {
      BooleanQuery bq = new BooleanQuery();
  
      // Set the synonym as a required phrase query.  Phrase queries handle multi-word synonyms, but require construction.
      String queryString = synonym.trim().toLowerCase();
      String[] parts = queryString.split("\\s+");
      PhraseQuery query = new PhraseQuery();
      Arrays.stream(parts).forEach(p -> query.add(new Term(field, p)));
      bq.add(query, BooleanClause.Occur.MUST);
  
      // Append all keywords as optional clauses.  The more of these we find, the higher the score will be.
      KEYWORDS.forEach(term -> bq.add(new TermQuery(new Term(field, term)), BooleanClause.Occur.SHOULD));
  
      return bq;
    }
  
    public static class SearchResult {
      String id;
      String title;
      Float relevanceScore;
  
      public SearchResult(String id, String title, Float relevanceScore) {
        this.id = id;
        this.title = title;
        /* Relevance scores are defined by Apache Lucene, and are dependent on the structure of the query.
         * See the scoring docs at
         * https://lucene.apache.org/core/5_2_1/core/org/apache/lucene/search/package-summary.html#package_description
         * for details. */
        this.relevanceScore = relevanceScore;
      }
  
      public String getId() {
        return id;
      }
  
      public String getTitle() {
        return title;
      }
  
      public Float getRelevanceScore() {
        return relevanceScore;
      }
    }
  }
  