content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.twentyn.patentScorer;
  
  import com.fasterxml.jackson.annotation.JsonAutoDetect;
  import com.fasterxml.jackson.annotation.JsonProperty;
  import com.fasterxml.jackson.annotation.PropertyAccessor;
  import com.fasterxml.jackson.databind.JavaType;
  import com.fasterxml.jackson.databind.ObjectMapper;
  import com.fasterxml.jackson.databind.SerializationConfig;
  import com.fasterxml.jackson.databind.SerializationFeature;
  import com.twentyn.patentSearch.DocumentSearch;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.DefaultParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  
  import java.io.BufferedReader;
  import java.io.CharArrayReader;
  import java.io.File;
  import java.io.FileReader;
  import java.io.FileWriter;
  import java.io.FilenameFilter;
  import java.nio.CharBuffer;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collections;
  import java.util.Comparator;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.regex.Pattern;
  
  public class ScoreMerger {
    public static final Logger LOGGER = LogManager.getLogger(ScoreMerger.class);
  
    public static void main(String[] args) throws Exception {
      System.out.println("Starting up...");
      System.out.flush();
      Options opts = new Options();
      opts.addOption(Option.builder("h").longOpt("help").desc("Print this help message and exit").build());
  
      opts.addOption(Option.builder("r").
          longOpt("results").required().hasArg().desc("A directory of search results to read").build());
      opts.addOption(Option.builder("s").
          longOpt("scores").required().hasArg().desc("A directory of patent classification scores to read").build());
      opts.addOption(Option.builder("o").
          longOpt("output").required().hasArg().desc("The output file where results will be written.").build());
  
      HelpFormatter helpFormatter = new HelpFormatter();
      CommandLineParser cmdLineParser = new DefaultParser();
      CommandLine cmdLine = null;
      try {
        cmdLine = cmdLineParser.parse(opts, args);
      } catch (ParseException e) {
        System.out.println("Caught exception when parsing command line: " + e.getMessage());
        helpFormatter.printHelp("DocumentIndexer", opts);
        System.exit(1);
      }
  
      if (cmdLine.hasOption("help")) {
        helpFormatter.printHelp("DocumentIndexer", opts);
        System.exit(0);
      }
      File scoresDirectory = new File(cmdLine.getOptionValue("scores"));
      if (cmdLine.getOptionValue("scores") == null || !scoresDirectory.isDirectory()) {
        LOGGER.error("Not a directory of score files: " + cmdLine.getOptionValue("scores"));
      }
  
      File resultsDirectory = new File(cmdLine.getOptionValue("results"));
      if (cmdLine.getOptionValue("results") == null || !resultsDirectory.isDirectory()) {
        LOGGER.error("Not a directory of results files: " + cmdLine.getOptionValue("results"));
      }
  
      FileWriter outputWriter = new FileWriter(cmdLine.getOptionValue("output"));
  
      ObjectMapper objectMapper = new ObjectMapper();
      objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
      objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
  
      FilenameFilter jsonFilter = new FilenameFilter() {
        public final Pattern JSON_PATTERN = Pattern.compile("\\.json$");
  
        public boolean accept(File dir, String name) {
          return JSON_PATTERN.matcher(name).find();
        }
      };
  
      Map<String, PatentScorer.ClassificationResult> scores = new HashMap<>();
      LOGGER.info("Reading scores from directory at " + scoresDirectory.getAbsolutePath());
      for (File scoreFile : scoresDirectory.listFiles(jsonFilter)) {
        BufferedReader reader = new BufferedReader(new FileReader(scoreFile));
        int count = 0;
        String line;
        while ((line = reader.readLine()) != null) {
          PatentScorer.ClassificationResult res = objectMapper.readValue(line, PatentScorer.ClassificationResult.class);
          scores.put(res.docId, res);
          count++;
        }
        LOGGER.info("Read " + count + " scores from " + scoreFile.getAbsolutePath());
      }
  
      Map<String, List<DocumentSearch.SearchResult>> synonymsToResults = new HashMap<>();
      Map<String, List<DocumentSearch.SearchResult>> inchisToResults = new HashMap<>();
      LOGGER.info("Reading results from directory at " + resultsDirectory);
      // With help from http://stackoverflow.com/questions/6846244/jackson-and-generic-type-reference.
      JavaType resultsType =
          objectMapper.getTypeFactory().constructCollectionType(List.class, DocumentSearch.SearchResult.class);
  
      List<File> resultsFiles = Arrays.asList(resultsDirectory.listFiles(jsonFilter));
      Collections.sort(resultsFiles, new Comparator<File>() {
        @Override
        public int compare(File o1, File o2) {
          return o1.getName().compareTo(o2.getName());
        }
      });
      for (File resultsFile : resultsFiles) {
        BufferedReader reader = new BufferedReader(new FileReader(resultsFile));
        CharBuffer buffer = CharBuffer.allocate(Long.valueOf(resultsFile.length()).intValue());
        int bytesRead = reader.read(buffer);
        LOGGER.info("Read " + bytesRead + " bytes from " + resultsFile.getName() +
            " (length is " + resultsFile.length() + ")");
        List<DocumentSearch.SearchResult> results =
            objectMapper.readValue(new CharArrayReader(buffer.array()), resultsType);
  
        LOGGER.info("Read " + results.size() + " results from " + resultsFile.getAbsolutePath());
  
        int count = 0;
        for (DocumentSearch.SearchResult sres : results) {
          for (DocumentSearch.ResultDocument resDoc : sres.getResults()) {
            String docId = resDoc.getDocId();
            PatentScorer.ClassificationResult classificationResult = scores.get(docId);
            if (classificationResult == null) {
              LOGGER.warn("No classification result found for " + docId);
            } else {
              resDoc.setClassifierScore(classificationResult.getScore());
            }
          }
          if (!synonymsToResults.containsKey(sres.getSynonym())) {
            synonymsToResults.put(sres.getSynonym(), new ArrayList<DocumentSearch.SearchResult>());
          }
          synonymsToResults.get(sres.getSynonym()).add(sres);
          count++;
          if (count % 1000 == 0) {
            LOGGER.info("Processed " + count + " search result documents");
          }
        }
      }
  
      Comparator<DocumentSearch.ResultDocument> resultDocumentComparator =
          new Comparator<DocumentSearch.ResultDocument>() {
            @Override
            public int compare(DocumentSearch.ResultDocument o1, DocumentSearch.ResultDocument o2) {
              int cmp = o2.getClassifierScore().compareTo(o1.getClassifierScore());
              if (cmp != 0) {
                return cmp;
              }
              cmp = o2.getScore().compareTo(o1.getScore());
              return cmp;
            }
          };
  
      for (Map.Entry<String, List<DocumentSearch.SearchResult>> entry : synonymsToResults.entrySet()) {
        DocumentSearch.SearchResult newSearchRes = null;
        // Merge all result documents into a single search result.
        for (DocumentSearch.SearchResult sr : entry.getValue()) {
          if (newSearchRes == null) {
            newSearchRes = sr;
          } else {
            newSearchRes.getResults().addAll(sr.getResults());
          }
        }
        if (newSearchRes == null || newSearchRes.getResults() == null) {
          LOGGER.error("Search results for " + entry.getKey() + " are null.");
          continue;
        }
        Collections.sort(newSearchRes.getResults(), resultDocumentComparator);
        if (!inchisToResults.containsKey(newSearchRes.getInchi())) {
          inchisToResults.put(newSearchRes.getInchi(), new ArrayList<DocumentSearch.SearchResult>());
        }
        inchisToResults.get(newSearchRes.getInchi()).add(newSearchRes);
      }
  
      List<String> sortedKeys = new ArrayList<String>(inchisToResults.keySet());
      Collections.sort(sortedKeys);
      List<GroupedInchiResults> orderedResults = new ArrayList<>(sortedKeys.size());
      Comparator<DocumentSearch.SearchResult> synonymSorter = new Comparator<DocumentSearch.SearchResult>() {
        @Override
        public int compare(DocumentSearch.SearchResult o1, DocumentSearch.SearchResult o2) {
          return o1.getSynonym().compareTo(o2.getSynonym());
        }
      };
      for (String inchi : sortedKeys) {
        List<DocumentSearch.SearchResult> res = inchisToResults.get(inchi);
        Collections.sort(res, synonymSorter);
        orderedResults.add(new GroupedInchiResults(inchi, res));
      }
  
      objectMapper.writerWithView(Object.class).writeValue(outputWriter, orderedResults);
      outputWriter.close();
    }
  
    public static class GroupedInchiResults {
      @JsonProperty("inchi")
      protected String inchi;
      @JsonProperty("search_results")
      protected List<DocumentSearch.SearchResult> results;
  
      protected GroupedInchiResults() {
      }
  
      public GroupedInchiResults(String inchi, List<DocumentSearch.SearchResult> results) {
        this.inchi = inchi;
        this.results = results;
      }
  
      public String getInchi() {
        return inchi;
      }
  
      public List<DocumentSearch.SearchResult> getResults() {
        return results;
      }
    }
  }
  