content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package act.installer.wikipedia;
  
  import java.io.BufferedReader;
  import java.io.BufferedWriter;
  import java.io.File;
  import java.io.FileReader;
  import java.io.FileWriter;
  import java.io.IOException;
  import java.io.Serializable;
  import java.util.HashSet;
  import java.util.regex.Pattern;
  import java.util.regex.Matcher;
  import java.util.Arrays;
  import java.util.Set;
  import java.util.ArrayList;
  import java.util.List;
  import org.apache.commons.lang3.StringUtils;
  
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.DefaultParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  
  import chemaxon.formats.MolFormatException;
  import chemaxon.formats.MolImporter;
  import com.fasterxml.jackson.annotation.JsonProperty;
  import com.fasterxml.jackson.databind.ObjectMapper;
  import org.apache.commons.csv.CSVFormat;
  import org.apache.commons.csv.CSVPrinter;
  
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  
  /**
   * This class parses Wikipedia data dumps to extract important chemicals. When called from the command line, it exports
   * an important chemicals wikipedia file to be used by the Installer database.
   * Usage:
   * sbt 'runMain act.installer.wikipedia.ImportantChemicalsWikipedia
   *                 -i data/enwiki-20160501-pages-articles.xml
   *                 -o MNT_SHARED_DATA/Thomas/imp_chemicals_wikipedia.txt
   *                 -t'
   */
  
  
  public class ImportantChemicalsWikipedia {
  
    private static final Logger LOGGER = LogManager.getFormatterLogger(ImportantChemicalsWikipedia.class);
    public static final CSVFormat TSV_FORMAT = CSVFormat.newFormat('\t').
        withRecordSeparator('\n').withIgnoreEmptyLines(true).withCommentMarker('#');
  
    public static final String OPTION_WIKIPEDIA_DUMP_FULL_PATH = "i";
    public static final String OPTION_OUTPUT_PATH = "o";
    public static final String OPTION_TSV_OUTPUT = "t";
  
    public static final String HELP_MESSAGE = StringUtils.join(new String[]{
        "This class parses Wikipedia data dumps to extract important chemicals."
    }, "");
  
    public static final List<Option.Builder> OPTION_BUILDERS = new ArrayList<Option.Builder>() {{
      add(Option.builder(OPTION_WIKIPEDIA_DUMP_FULL_PATH)
          .argName("WIKIPEDIA_DUMP_PATH")
          .desc("The full path to the Wikipedia XML dump to parse. It should be located on the NAS " +
              "(data/enwiki-20160501-pages-articles.xml) but can also be obtained from " +
              "https://dumps.wikimedia.org/enwiki/")
          .hasArg().required()
          .longOpt("wikipedia_dump_path")
          .type(String.class)
      );
      add(Option.builder(OPTION_OUTPUT_PATH)
          .argName("OUTPUT_PATH")
          .desc("The full path to write the output data.")
          .hasArg().required()
          .longOpt("output_path")
          .type(String.class)
      );
      add(Option.builder(OPTION_TSV_OUTPUT)
          .argName("TSV_OUTPUT")
          .desc("Whether the output should be written in TSV format.")
          .longOpt("tsv")
          .type(boolean.class)
      );
      add(Option.builder("h")
          .argName("help")
          .desc("Prints this help message")
          .longOpt("help")
      );
    }};
  
    public static final HelpFormatter HELP_FORMATTER = new HelpFormatter();
  
    static {
      HELP_FORMATTER.setWidth(100);
    }
  
    private static final String DATABASE_TYPE = "WIKIPEDIA";
  
    // Some Wikipedia pages contains InChI strings but are not about a specific Chemical.
    // A good heuristic to exclude them is to list words that appear in the titles.
    // A title is considered "valid" if it does not include any of these strings.
    private static final String[] EXCLUDE_TITLES_WITH_WORDS_LIST =
        new String[] {"Identifier", "Wikipedia", "InChI", "Template", "testcase"};
    private static final Set<String> EXCLUDE_TITLES_WITH_WORDS = new HashSet<>(
        Arrays.asList(EXCLUDE_TITLES_WITH_WORDS_LIST));
  
    // Some InChI cause fatal Java errors when trying to validate them through Chemaxon's library. Ignore them.
    // There does not seem to exist a more elegant way to do this.
    private static final String[] EXCLUDE_INCHIS_LIST =
        new String[] {"InChI = 1/C12H10AsCl/c14/h1-10H"};
    private static final Set<String> EXCLUDE_INCHIS = new HashSet<>(
        Arrays.asList(EXCLUDE_INCHIS_LIST));
  
    // These patterns allow to identify Wikipedia titles and InChIs.
    private static final Pattern TITLE_PATTERN = Pattern.compile(".*<title>([^<>]+)</title>.*");
    private static final Pattern INCHI_PATTERN =
        Pattern.compile(".*(?i)(InChI[0-9]?\\p{Space}*=\\p{Space}*1S?/[\\p{Space}0-9a-z+\\-\\(\\)/.,\\?;\\*]+).*");
  
    private static ObjectMapper mapper = new ObjectMapper();
  
    private String lastTitle;
    private boolean isLastTitleValid;
    private static HashSet<ImportantChemical> importantChemicalsWikipedia = new HashSet<>();
  
    public ImportantChemicalsWikipedia() {}
  
    public class ImportantChemical implements Serializable {
  
      @JsonProperty("type")
      private String type;
  
      @JsonProperty("dbid")
      private String dbid;
  
      @JsonProperty("inchi")
      private String inchi;
  
      @JsonProperty("metadata")
      private WikipediaMetadata metadata;
  
      public ImportantChemical(String type, String dbid, String inchi, WikipediaMetadata metadata) {
        this.type = type;
        this.dbid = dbid;
        this.inchi = inchi;
        this.metadata = metadata;
      }
  
      public String getType() {
        return type;
      }
  
      public String getDbid() {
        return dbid;
      }
  
      public String getInchi() {
        return inchi;
      }
  
      public WikipediaMetadata getMetadata() {
        return metadata;
      }
    }
  
    public class WikipediaMetadata {
  
      @JsonProperty("article")
      private String article;
  
      @JsonProperty("std_inchi")
      private boolean stdInChI;
  
      public WikipediaMetadata(String article, boolean stdInChI) {
        this.article = article;
        this.stdInChI = stdInChI;
      }
    }
  
    /**
     * This function extracts an InChI string from a candidate line.
     * @param line a String from the raw XML data source file
     * @return a String representing the molecule's InChI
     */
    public String extractInchiFromLine(String line) {
      Matcher inchiMatcher = INCHI_PATTERN.matcher(line);
      if (inchiMatcher.matches()) {
        return inchiMatcher.group(1);
      }
      return null;
    }
  
    /**
     * This function formats a matched InChI to make it canonical.
     * @param inchi a String representing the molecule's InChI
     * @return a formatted string representing the corresponding canonical InChI
     */
    public String formatInchiString(String inchi) {
      // Remove all whitespaces
      String tmpInchi = inchi.replaceAll("\\s+","");
  
      // Some InChIs start with "InChI1" or "InChI2". We need to remove the suffix ("1", "2") to allow Chemaxon validation
      String formattedInchi = tmpInchi.replaceAll("InChI[0-9]?", "InChI");
  
      return formattedInchi;
    }
  
    /**
     * This function tries to import a molecule in Chemaxon and returns a boolean indicating whether or not it succeeded.
     * @param inchi a string representing the molecule's canonical InChI
     * @return a boolean indicating success or failure to import the molecule in Chemaxon
     */
    public boolean isChemaxonValidInchi(String inchi) {
      try {
        MolImporter.importMol(inchi);
      } catch (MolFormatException e) {
        return false;
      }
      return true;
    }
  
    /**
     * This function processes a line found to contain a candidate InChI and adds potential candidate molecules to the
     * important chemicals set.
     * @param line a String from the raw XML data source file
     */
    public void processInchiLine(String line) throws IOException {
  
      String inchi;
  
      // Extract a potential Inchi from the line. Check if null.
      if ((inchi = extractInchiFromLine(line)) != null) {
        if (!EXCLUDE_INCHIS.contains(inchi)) {
  
          // InChI formatting
          String formattedInchi = formatInchiString(inchi);
          LOGGER.trace(formattedInchi);
  
          // InChI validation through Chemaxon library
          boolean isChemaxonValidInchi = isChemaxonValidInchi(formattedInchi);
          if (!isChemaxonValidInchi) {
            LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.info("Chemaxon validation failed");
            LOGGER.info("Last title      : %s", lastTitle);
            LOGGER.info("Extracted line  : %s", line);
            LOGGER.info("Matched InChI   : %s", inchi);
            LOGGER.info("Formatted InChI : %s", formattedInchi);
          } else {
            boolean isStandardInchi = formattedInchi.startsWith("InChI=1S");
            String wikipediaURL = "https://en.wikipedia.org/wiki/" + lastTitle.replace(" ", "_");
  
            WikipediaMetadata metadata = new WikipediaMetadata(lastTitle, isStandardInchi);
  
            ImportantChemical importantChemical = new ImportantChemical(
                DATABASE_TYPE , wikipediaURL, formattedInchi, metadata);
            importantChemicalsWikipedia.add(importantChemical);
          }
        }
      }
    }
  
    /**
     * This function processes a line from the data source to find titles or InChIs
     * @param line a String from the raw XML data source file
     */
    public void processLine(String line) throws IOException {
      Matcher titleMatcher = TITLE_PATTERN.matcher(line);
  
      if (titleMatcher.matches()) {
        lastTitle = titleMatcher.group(1);
        isLastTitleValid = true;
        for (String excludedWord : EXCLUDE_TITLES_WITH_WORDS) {
          if (lastTitle.contains(excludedWord)) {
            isLastTitleValid = false;
          }
        }
      } else {
        if (isLastTitleValid) {
          String lowerCaseLine = line.toLowerCase();
          if (lowerCaseLine.contains("inchi") && !lowerCaseLine.contains("inchikey")
              && !lowerCaseLine.contains("inchi_ref")) {
            processInchiLine(line);
          }
        }
      }
    }
  
    /**
     * This function writes the important chemicals set to a TSV file.
     * @param outputPath a String indicating where the file should be written (including its name)
     */
    public void writeToTSV(String outputPath) {
      try {
        BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath));
        CSVPrinter printer = new CSVPrinter(writer, TSV_FORMAT);
        printer.printComment("This file has been generated by the ImportantChemicalsWikipedia.java script.");
        printer.printComment("Format: WIKIPEDIA<tab><wikipedia url><tab><inchi><tab><metadata>");
        for (ImportantChemical importantChemical : importantChemicalsWikipedia) {
          List<String> nextLine = new ArrayList<>();
          nextLine.add(importantChemical.getType());
          nextLine.add(importantChemical.getDbid());
          nextLine.add(importantChemical.getInchi());
          nextLine.add(mapper.writeValueAsString(importantChemical.getMetadata()));
          printer.printRecord(nextLine);
        }
        printer.flush();
        writer.close();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  
    /**
     * This function writes the important chemicals set to a JSON file.
     * @param outputPath a String indicating where the file should be written (including its name)
     */
    public void writeToJSON(String outputPath) throws IOException {
      File file = new File(outputPath);
      mapper.writeValue(file, importantChemicalsWikipedia);
    }
  
  
    public static void main(final String[] args) throws IOException {
  
      Options opts = new Options();
      for (Option.Builder b : OPTION_BUILDERS) {
        opts.addOption(b.build());
      }
  
      CommandLine cl = null;
      try {
        CommandLineParser parser = new DefaultParser();
        cl = parser.parse(opts, args);
      } catch (ParseException e) {
        System.err.format("Argument parsing failed: %s\n", e.getMessage());
        HELP_FORMATTER.printHelp(ImportantChemicalsWikipedia.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        System.exit(1);
      }
  
      if (cl.hasOption("help")) {
        HELP_FORMATTER.printHelp(ImportantChemicalsWikipedia.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        return;
      }
  
      String inputPath = cl.getOptionValue(OPTION_WIKIPEDIA_DUMP_FULL_PATH, "1000");
      String outputPath = cl.getOptionValue(OPTION_OUTPUT_PATH, "1000");
      Boolean outputTSV = cl.hasOption(OPTION_TSV_OUTPUT);
  
      ImportantChemicalsWikipedia importantChemicalsWikipedia = new ImportantChemicalsWikipedia();
  
      try (BufferedReader br = new BufferedReader(new FileReader(inputPath))) {
        String line;
        while ((line = br.readLine()) != null) {
          importantChemicalsWikipedia.processLine(line);
        }
      }
      catch (IOException e) {
        LOGGER.error(e);
      }
  
      if (outputTSV) {
        importantChemicalsWikipedia.writeToTSV(outputPath);
      } else {
        importantChemicalsWikipedia.writeToJSON(outputPath);
      }
    }
  }
  