content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package act.installer.bing;
  
  import act.server.BingCacheMongoDB;
  import com.fasterxml.jackson.databind.JsonNode;
  import com.fasterxml.jackson.databind.ObjectMapper;
  import com.mongodb.BasicDBList;
  import com.mongodb.BasicDBObject;
  import org.apache.http.HttpEntity;
  import org.apache.http.client.methods.CloseableHttpResponse;
  import org.apache.http.client.methods.HttpGet;
  import org.apache.http.client.utils.URIBuilder;
  import org.apache.http.entity.ContentType;
  import org.apache.http.impl.client.CloseableHttpClient;
  import org.apache.http.impl.client.HttpClients;
  import org.apache.http.HttpStatus;
  import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
  import org.apache.logging.log4j.LogManager;
  import org.apache.logging.log4j.Logger;
  
  import java.io.BufferedReader;
  import java.io.FileInputStream;
  import java.io.InputStreamReader;
  import java.io.IOException;
  import java.net.URI;
  import java.net.URISyntaxException;
  import java.net.URLEncoder;
  import java.nio.charset.Charset;
  import java.nio.charset.StandardCharsets;
  import java.util.HashSet;
  import java.util.Set;
  
  /**
   * BingSearchResults provides methods for:
   * - querying the Bing Search API,
   * - caching its results in a Mongo database
   * - returning searched or cached results
   * - finding the best name for a molecule
   */
  
  public class BingSearchResults {
  
    private static final Logger LOGGER = LogManager.getFormatterLogger(BingSearchResults.class);
  
    // Full path to the account key for the Bing Search API (on the NAS)
    private static final String ACCOUNT_KEY_FILEPATH = "data/bing/bing_search_api_account_key.txt";
    // Maximum number of results possible per API call. This is the maximum value for URL parameter "count"
    private static final Integer MAX_RESULTS_PER_CALL = 50;
    // How many search results should be retrieved when getting topSearchResults
    private static final Integer TOP_N = 50;
    
    // The centralized location for caching Bing Search queries.
    // TODO: make this changeable without a code change (with CLI maybe?)
    private static final String BING_CACHE_HOST = "localhost";
    private static final int BING_CACHE_MONGO_PORT = 27777;
    private static final String BING_CACHE_MONGO_DATABASE = "bingsearch";
  
  
    private static final String BING_API_HOST = "api.cognitive.microsoft.com";
    private static final String BING_API_PATH = "/bing/v5.0/search";
  
    private static ObjectMapper mapper = new ObjectMapper();
  
    private BingCacheMongoDB bingCacheMongoDB;
    private BasicHttpClientConnectionManager basicConnManager;
    private String accountKey;
    private boolean cacheOnly;
  
    public BingSearchResults() {
      this(ACCOUNT_KEY_FILEPATH);
    }
  
    public BingSearchResults(boolean cacheOnly) {
      this.cacheOnly = cacheOnly;
      this.bingCacheMongoDB = new BingCacheMongoDB(BING_CACHE_HOST, BING_CACHE_MONGO_PORT, BING_CACHE_MONGO_DATABASE);
    }
  
    public BingSearchResults(String accountKeyFilepath) {
      this.cacheOnly = false;
      this.bingCacheMongoDB = new BingCacheMongoDB(BING_CACHE_HOST, BING_CACHE_MONGO_PORT, BING_CACHE_MONGO_DATABASE);
      this.basicConnManager = new BasicHttpClientConnectionManager();
      try {
        this.accountKey = getAccountKey(accountKeyFilepath);
      } catch (IOException e) {
        String msg = String.format("Bing Searcher could not find account key at %s", accountKeyFilepath);
        LOGGER.error(msg);
        throw new RuntimeException(msg);
      }
    }
  
    /** This function gets the account key located on the NAS
     * @return the account key to be used for authentication purposes
     * @throws IOException
     */
    private static String getAccountKey(String accountKeyFilename) throws IOException {
      FileInputStream fs = new FileInputStream(accountKeyFilename);
      BufferedReader br = new BufferedReader(new InputStreamReader(fs));
      String account_key = br.readLine();
      return account_key;
    }
  
    /** This function fetches the total number of Bing search results and return the "totalCountSearchResult".
     * @param formattedName name that will be used as search query, lowercase formatted
     * @return the total count search results from Bing search
     * @throws IOException
     */
    private Long fetchTotalCountSearchResults(String formattedName) throws IOException {
      LOGGER.debug("Updating totalCountSearchResults for name: %s.", formattedName);
      final String queryTerm = URLEncoder.encode(formattedName, StandardCharsets.UTF_8.name());
      // Set count to 1 and offset to 0 since we need only one search result to extract the estimated count.
      final int count = 1;
      final int offset = 0;
      JsonNode results = fetchBingSearchAPIResponse(queryTerm, count, offset);
      return results.path("totalEstimatedMatches").asLong();
    }
  
  
    /** This function fetches the topN Bing search results for the current instance of NameSearchResult object
     * and updates the "topSearchResults" instance variable. Existing value is overridden.
     * @param formattedName name that will be used as search query, lowercase formatted
     * @param topN number of Web results to fetch from Bing Search API
     * @return returns a set of SearchResults containing the topN Bing search results
     * @throws IOException
     */
    private Set<SearchResult> fetchTopSearchResults(String formattedName, Integer topN)
        throws IOException {
      LOGGER.debug("Updating topSearchResults for name: %s.", formattedName);
      Set<SearchResult> topSearchResults = new HashSet<>();
      final String queryTerm = URLEncoder.encode(formattedName, StandardCharsets.UTF_8.name());
      // The Bing search API cannot return more than 100 results at once, but it is possible to iterate
      // through the results.
      // For example, if we need topN = 230 results, we will issue the following queries
      // (count and offset are URL parameters)
      // QUERY 1: count = 100, offset = 0
      // QUERY 2: count = 100, offset = 100
      // QUERY 3: count = 30, offset = 200
      Integer iterations = topN / MAX_RESULTS_PER_CALL;
      Integer remainder = topN % MAX_RESULTS_PER_CALL;
      for (int i = 0; i < iterations; i++) {
        topSearchResults.addAll(fetchSearchResults(queryTerm, MAX_RESULTS_PER_CALL, MAX_RESULTS_PER_CALL * i));
      }
      if (remainder > 0) {
        topSearchResults.addAll(fetchSearchResults(queryTerm, remainder, MAX_RESULTS_PER_CALL * iterations));
      }
      return topSearchResults;
    }
  
  
    /** This function issues a Bing Search API call and parses the response to extract a set of SearchResults.
     * @param query (String) the term to query for.
     * @param count (int) URL parameter indicating how many results to return. Max value is 100.
     * @param offset (int) URL parameter indicating the offset for results.
     * @return returns a set of SearchResults containing [count] search results with offset [offset]
     * @throws IOException
     */
    private Set<SearchResult> fetchSearchResults(String query, int count, int offset) throws IOException {
      if (count > MAX_RESULTS_PER_CALL) {
        LOGGER.warn("Number of results requested (%d) was too high. Will get only %d", count, MAX_RESULTS_PER_CALL);
      }
      Set<SearchResult> searchResults = new HashSet<>();
      JsonNode results = fetchBingSearchAPIResponse(query, count, offset);
      final JsonNode webResults = results.path("value");
      for (JsonNode webResult : webResults) {
        SearchResult searchResult = new SearchResult();
        searchResult.populateFromJsonNode(webResult);
        searchResults.add(searchResult);
      }
      return searchResults;
    }
  
    /** This function issues a Bing search API call and gets the JSONObject containing the relevant results
     * (including TotalCounts and SearchResults)
     * @param queryTerm (String) the term to query for.
     * @param count (int) URL parameter indicating how many results to return. Max value is 100.
     * @param offset (int) URL parameter indicating the offset for results.
     * @return a JSONObject containing the response.
     * @throws IOException
     */
    private JsonNode fetchBingSearchAPIResponse(String queryTerm, Integer count, Integer offset) throws IOException {
  
      if (count <= 0) {
        LOGGER.error("Bing Search API was called with \"count\" URL parameter = 0. Please request at least one result.");
        return null;
      }
  
      URI uri = null;
      try {
        // Bing URL pattern. Note that we use composite queries to allow retrieval of the total results count.
        // Transaction cost is [count] bings, where [count] is the value of the URL parameter "count".
        // In other words, we can make 5M calls with [count]=1 per month.
  
  
        // Example: https://api.cognitive.microsoft.com/bing/v5.0/search?q=porsche&responseFilter=webpages
        uri = new URIBuilder()
            .setScheme("https")
            .setHost(BING_API_HOST)
            .setPath(BING_API_PATH)
            // Wrap the query term (%s) with double quotes (%%22) for exact search
            .setParameter("q", String.format("%s", queryTerm))
            // Restrict response to Web Pages only
            .setParameter("responseFilter", "webpages")
            // "count" parameter.
            .setParameter("count", count.toString())
            // "offset" parameter.
            .setParameter("offset", offset.toString())
            .build();
  
      } catch (URISyntaxException e) {
        LOGGER.error("An error occurred when trying to build the Bing Search API URI", e);
      }
  
      JsonNode results;
      HttpGet httpget = new HttpGet(uri);
      // Yay for un-encrypted account key!
      // TODO: actually is there a way to encrypt it?
      httpget.setHeader("Ocp-Apim-Subscription-Key", accountKey);
  
      CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(basicConnManager).build();
  
      try (CloseableHttpResponse response = httpclient.execute(httpget)) {
        Integer statusCode = response.getStatusLine().getStatusCode();
  
        // TODO: The Web Search API returns useful error messages, we could use them to have better insights on failures.
        // See: https://dev.cognitive.microsoft.com/docs/services/56b43eeccf5ff8098cef3807/operations/56b4447dcf5ff8098cef380d
        if (!statusCode.equals(HttpStatus.SC_OK)) {
          LOGGER.error("Bing Search API call returned an unexpected status code (%d) for URI: %s", statusCode, uri);
          return null;
        }
  
        HttpEntity entity = response.getEntity();
        ContentType contentType = ContentType.getOrDefault(entity);
        Charset charset = contentType.getCharset();
        if (charset == null) {
          charset = StandardCharsets.UTF_8;
        }
  
        try (final BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) {
          String inputLine;
          final StringBuilder stringResponse = new StringBuilder();
          while ((inputLine = in.readLine()) != null) {
            stringResponse.append(inputLine);
          }
          JsonNode rootNode = mapper.readValue(stringResponse.toString(), JsonNode.class);
          results = rootNode.path("webPages");
        }
      }
      return results;
    }
  
    /** This key function caches in a MongoDB collection and returns a set of SearchResults.
     * If present, the results are returned from the cache. If not, the results are queried and returned after updating
     * the cache.
     * @param name (String) the name to return results for. Will be normalized to lower case.
     * @return a set of SearchResults
     * @throws IOException
     */
    public Set<SearchResult> getAndCacheTopSearchResults(String name) throws IOException {
  
      String formattedName = name.toLowerCase();
      BasicDBObject nameSearchResultDBObject = bingCacheMongoDB.getNameSearchResultDBObjectFromName(formattedName);
      Set<SearchResult> searchResults = new HashSet<>();
  
      // There are 3 cases:
      // 1) There is a corresponding entry in the cache AND the topSearchResults are populated.
      //    In this case, we read from the cache and return the results.
      // 2) There is a corresponding entry in the cache BUT the topSearchResults are not populated.
      //    This case occurs when only totalCountSearchResults is populated.
      //    In this case, perform the relevant query, update the cache and return the results
      // 3) There is no corresponding entry in the cache.
      //    In this case, perform the relevant query, create a new entry in the cache and return the results.
  
      if (nameSearchResultDBObject == null) {
        // Case 3)
        LOGGER.debug("No corresponding entry in the cache. Fetching results and populating cache.");
        // Query the results
        searchResults = fetchTopSearchResults(formattedName, TOP_N);
        // Create new object and update it
        NameSearchResults nameSearchResults = new NameSearchResults(formattedName);
        nameSearchResults.setTopSearchResults(searchResults);
        // Save new document in the cache
        bingCacheMongoDB.cacheNameSearchResult(nameSearchResults);
        return searchResults;
      }
  
      // There is an existing entry in the DB
      BasicDBList topSearchResultsList = (BasicDBList) nameSearchResultDBObject.get("topSearchResults");
      if (topSearchResultsList == null) {
        // Case 2)
        LOGGER.debug("Existing entry in the cache, with empty topSearchResults. Fetching results and updating cache.");
        // Query the results
        searchResults = fetchTopSearchResults(formattedName, TOP_N);
        // Create new object and update its instance variable
        NameSearchResults nameSearchResults = new NameSearchResults(formattedName);
        nameSearchResults.setTopSearchResults(searchResults);
        // Update the cache
        bingCacheMongoDB.updateTopSearchResults(formattedName, nameSearchResults);
        return searchResults;
      }
  
      // Case 1)
      LOGGER.debug("Existing entry in the cache, with populated topSearchResults. Returning from the cache.");
      for (Object topSearchResult : topSearchResultsList) {
        SearchResult searchResult = new SearchResult();
        BasicDBObject topSearchResultDBObject = (BasicDBObject) topSearchResult;
        searchResult.populateFromBasicDBObject(topSearchResultDBObject);
        searchResults.add(searchResult);
      }
      return searchResults;
    }
  
    public Set<SearchResult> getTopSearchResultsFromCache(String name) {
      Set<SearchResult> searchResults = new HashSet<>();
      String formattedName = name.toLowerCase();
      BasicDBObject nameSearchResultDBObject = bingCacheMongoDB.getNameSearchResultDBObjectFromName(formattedName);
      if (nameSearchResultDBObject == null) {
        return searchResults;
      }
      BasicDBList topSearchResultsList = (BasicDBList) nameSearchResultDBObject.get("topSearchResults");
      if (topSearchResultsList == null) {
        return searchResults;
      }
      for (Object topSearchResult : topSearchResultsList) {
        SearchResult searchResult = new SearchResult();
        BasicDBObject topSearchResultDBObject = (BasicDBObject) topSearchResult;
        searchResult.populateFromBasicDBObject(topSearchResultDBObject);
        searchResults.add(searchResult);
      }
      return searchResults;
    }
  
    public Long getTotalCountSearchResultsFromCache(String name) {
      String formattedName = name.toLowerCase();
      BasicDBObject nameSearchResultDBObject = bingCacheMongoDB.getNameSearchResultDBObjectFromName(formattedName);
      Long totalCountSearchResults;
      if (nameSearchResultDBObject == null) {
        return -1L;
      }
      totalCountSearchResults = (Long) nameSearchResultDBObject.get("totalCountSearchResults");
      if (totalCountSearchResults == null) {
        return -1L;
      }
      return totalCountSearchResults;
    }
  
    /** This key function caches in a MongoDB collection and returns the total count of Bing search results.
     * If present, the results are returned from the cache. If not, the results are queried and returned after updating
     * the cache.
     * @param name (String) the name to return results for. Will be normalized to lower case.
     * @return the total search result count
     * @throws IOException
     */
    public Long getAndCacheTotalCountSearchResults(String name) throws IOException {
  
      String formattedName = name.toLowerCase();
      BasicDBObject nameSearchResultDBObject = bingCacheMongoDB.getNameSearchResultDBObjectFromName(formattedName);
      Long totalCountSearchResults;
  
      // There are 3 cases:
      // 1) There is a corresponding entry in the cache AND the totalCountSearchResults are populated.
      //    In this case, we read from the cache and return the results.
      // 2) There is a corresponding entry in the cache BUT the totalCountSearchResults are not populated.
      //    This case occurs when only topSearchResults is populated.
      //    In this case, perform the relevant query, update the cache and return the total count
      // 3) There is no corresponding entry in the cache.
      //    In this case, perform the relevant query, create a new entry in the cache and return the total count.
  
      if (nameSearchResultDBObject == null) {
        // Case 3)
        LOGGER.debug("No corresponding entry in the cache. Fetching results and populating cache.");
        // Query the results
        totalCountSearchResults = fetchTotalCountSearchResults(formattedName);
        // Create new object and update it
        NameSearchResults nameSearchResults = new NameSearchResults(formattedName);
        nameSearchResults.setTotalCountSearchResults(totalCountSearchResults);
        // Save new document in the cache
        bingCacheMongoDB.cacheNameSearchResult(nameSearchResults);
        return totalCountSearchResults;
      }
  
      // There is an existing entry in the cache
      totalCountSearchResults = (Long) nameSearchResultDBObject.get("totalCountSearchResults");
  
      if (totalCountSearchResults == null || totalCountSearchResults < 0) {
        // Case 2)
        LOGGER.debug("Existing entry in the cache, with empty totalCountSearchResults. " +
            "Fetching results and updating cache.");
        // Query the results
        totalCountSearchResults = fetchTotalCountSearchResults(formattedName);
        // Create new object and update its instance variable
        NameSearchResults nameSearchResults = new NameSearchResults(formattedName);
        nameSearchResults.setTotalCountSearchResults(totalCountSearchResults);
        // Update the cache
        bingCacheMongoDB.updateTotalCountSearchResults(formattedName, nameSearchResults);
        return totalCountSearchResults;
      }
  
      // Case 1)
      LOGGER.debug("Existing entry in the cache, with populated totalCountSearchResults. Returning from the cache.");
      return totalCountSearchResults;
    }
  
    /** Heuristic to find the best name for a given InChI, based on the total number of search results
     * @param namesOfMolecule (NamesOfMolecule) Java object containing Brenda, MetaCyc, ChEBI and DrugBank names for a given
     *                      InChI.
     * @return the name with the highest total number of search results, called Best Name
     * @throws IOException
     */
    public String findBestMoleculeName(NamesOfMolecule namesOfMolecule) throws IOException {
      Long maxCount = -1L;
      String bestName = "";
      String inchi = namesOfMolecule.getInchi();
      String[] splittedInchi = inchi.split("/");
      String formulaFromInchi = null;
      if (splittedInchi.length >= 2) {
        formulaFromInchi = inchi.split("/")[1];
      }
  
      LOGGER.debug("Formula %s extracted from %s", formulaFromInchi, inchi);
  
      String wikipediaName = namesOfMolecule.getWikipediaName();
      if (wikipediaName != null) {
        bestName = wikipediaName;
      } else {
        Set<String> names = namesOfMolecule.getAllNames();
        names.remove(formulaFromInchi);
        for (String name : names) {
          // Ignore name if <= 4 characters
          if (name.length() <= 4) {
            continue;
          }
          LOGGER.debug("Getting search hits for %s", name);
          Long count = (cacheOnly) ? getTotalCountSearchResultsFromCache(name) : getAndCacheTotalCountSearchResults(name);
          // Ignore name if there was a previous better candidate
          if (count <= maxCount) {
            continue;
          }
          maxCount = count;
          bestName = name;
        }
      }
  
      // Note we don't use ChEBI or DrugBank names to keep this function simple.
      // If Brenda and MetaCyc names are not populated, it is very rare that ChEBI or DrugBank would be.
      LOGGER.debug("Best name found for %s is %s", namesOfMolecule.getInchi(), bestName);
      return bestName;
    }
  
    public static void main(String[] args) {
      String apiKeyFilepath = "MNT_SHARED_DATA/Thomas/test-bing/microsoft-cognitive-service-api-key";
      BingSearchResults bingSearchResults = new BingSearchResults(apiKeyFilepath);
      try {
        Set<SearchResult> res = bingSearchResults.getAndCacheTopSearchResults("new query");
        Long count = bingSearchResults.getAndCacheTotalCountSearchResults("new query");
      } catch (IOException e) {
        throw new RuntimeException("Exception occurred when computing example query, %s", e);
      }
    }
  }
  