content: // This file is part of AceWiki.
  // Copyright 2008-2013, AceWiki developers.
  //
  // AceWiki is free software: you can redistribute it and/or modify it under the terms of the GNU
  // Lesser General Public License as published by the Free Software Foundation, either version 3 of
  // the License, or (at your option) any later version.
  //
  // AceWiki is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
  // even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  // Lesser General Public License for more details.
  //
  // You should have received a copy of the GNU Lesser General Public License along with AceWiki. If
  // not, see http://www.gnu.org/licenses/.
  
  package ch.uzh.ifi.attempto.acewiki.gf;
  
  import java.net.URI;
  import java.net.URISyntaxException;
  import java.util.Collections;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
  import java.util.Set;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import com.google.common.base.Functions;
  import com.google.common.base.Joiner;
  import com.google.common.base.Splitter;
  import com.google.common.collect.HashMultimap;
  import com.google.common.collect.ImmutableList;
  import com.google.common.collect.Maps;
  import com.google.common.collect.Multimap;
  import com.google.common.collect.Ordering;
  import com.google.common.collect.Sets;
  
  import ch.uzh.ifi.attempto.acewiki.core.Ontology;
  import ch.uzh.ifi.attempto.gfservice.GfModule;
  import ch.uzh.ifi.attempto.gfservice.GfParseResult;
  import ch.uzh.ifi.attempto.gfservice.GfService;
  import ch.uzh.ifi.attempto.gfservice.GfServiceException;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultBrowseAll;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultComplete;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultGrammar;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultLinearize;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultLinearizeAll;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultParse;
  import ch.uzh.ifi.attempto.gfservice.GfServiceResultRandom;
  import ch.uzh.ifi.attempto.gfservice.GfStorage;
  import ch.uzh.ifi.attempto.gfservice.GfStorageResult;
  import ch.uzh.ifi.attempto.gfservice.GfStorageResultLs;
  import ch.uzh.ifi.attempto.gfservice.gfwebservice.GfWebService;
  import ch.uzh.ifi.attempto.gfservice.gfwebservice.GfWebStorage;
  
  
  /**
   * This class wraps GF features of a particular GF grammar.
   *
   * TODO: move ACE-specific stuff out of this class
   *
   * @author Kaarel Kaljurand
   */
  public class GfGrammar {
  
  	// TODO: let the user configure the size of the ambiguity
  	public final static int GF_PARSE_LIMIT = 10;
  
  	private final int LINEARIZE_ALL_QUERY_LIMIT;
  
  	private final Logger mLogger = LoggerFactory.getLogger(GfGrammar.class);
  
  	// Some naming conventions
  	public final static String PREFIX_DISAMB = "Disamb";
  	public final static String SUFFIX_APE = "Ape";
  	public final static String EXTENSION_GF = ".gf";
  	public final static String EXTENSION_GFO = ".gfo";
  
  	// Note that true can remove (always removes?) lins
  	// which are not available in all the concretes,
  	// i.e. if you add a lin then you need to add it too all the concretes
  	// otherwise you cannot use it in a sentence.
  	private final static boolean OPTIMIZE_PGF = true;
  
  	private final static int GF_APE_FIELD_LOGICAL_SYMBOL = 3;
  
  	private final static char GF_TOKEN_SEPARATOR = ' ';
  	private final static char GF_TREE_SEPARATOR = '|';
  	private final static char GF_APE_SEPARATOR = '|';
  	private final static String GF_SERIALIZATION_SEPARATOR = "||";
  
  	public final static Joiner GF_TREE_JOINER = Joiner.on(GF_TREE_SEPARATOR);
  	public final static Joiner GF_SERIALIZATION_JOINER = Joiner.on(GF_SERIALIZATION_SEPARATOR).useForNull("");
  	public final static Joiner GF_TOKEN_JOINER = Joiner.on(GF_TOKEN_SEPARATOR);
  	public final static Splitter GF_TREE_SPLITTER = Splitter.on(GF_TREE_SEPARATOR).omitEmptyStrings();
  	public final static Splitter GF_APE_SPLITTER = Splitter.on(GF_APE_SEPARATOR);
  	public final static Splitter GF_SERIALIZATION_SPLITTER = Splitter.on(GF_SERIALIZATION_SEPARATOR);
  	public final static Splitter GF_TOKEN_SPLITTER = Splitter.on(GF_TOKEN_SEPARATOR);
  
  	private final GfService mGfService;
  	private final GfStorage mGfStorage;
  	private final String mCat;
  	private final String mDir;
  
  	private GfServiceResultGrammar mGfServiceResultGrammar;
  	private GfServiceResultBrowseAll mGfServiceResultBrowseAll;
  
  	private final Map<String, Multimap<String, String>> langToTokenToCats = Maps.newHashMap();
  	private final Map<String, Map<String, String>> langToIriToToken = Maps.newHashMap();
  
  	// TODO: could use a Multiset instead but there does not seem to be a
  	// short way to get out k-largest elements.
  	private final Map<String, Integer> mCatToSize = Maps.newHashMap();
  
  	public GfGrammar(Ontology ontology) {
  		URI serviceUri;
  		try {
  			serviceUri = new URI(ontology.getParameter("service_uri"));
  		} catch (URISyntaxException e) {
  			throw new RuntimeException(e);
  		}
  
  		String pgfName = ontology.getParameter("pgf_name");
  		mGfService = new GfWebService(serviceUri, pgfName);
  		mGfStorage = new GfWebStorage(serviceUri);
  		// Note: start_cat can be null, in this case the default start category is used
  		mCat = ontology.getParameter("start_cat");
  		mDir = getDir(pgfName);
  		LINEARIZE_ALL_QUERY_LIMIT = ontology.getParameterAsInt("linearize_all_query_limit");
  
  		try {
  			refreshGrammarInfo();
  			refreshLangToTokenToCats();
  		} catch (GfServiceException e) {
  			// TODO Auto-generated catch block
  			e.printStackTrace();
  		}
  	}
  
  
  	public GfServiceResultGrammar getGrammar() {
  		return mGfServiceResultGrammar;
  	}
  
  
  	/**
  	 * @return set of names of the concrete languages defined in the grammar
  	 */
  	public Set<String> getLanguages() {
  		if (mGfServiceResultGrammar == null) {
  			return Collections.emptySet();
  		}
  		return mGfServiceResultGrammar.getLanguages().keySet();
  	}
  
  
  	/**
  	 * @return set of locales defined for the given language in the grammar
  	 */
  	public Set<String> getLocales(String lang) {
  		if (mGfServiceResultGrammar == null) {
  			return Collections.emptySet();
  		}
  		return mGfServiceResultGrammar.getLanguages().get(lang);
  	}
  
  
  	/**
  	 * @return {@code true} iff the given grammar contains a concrete language with suffix SUFFIX_APE
  	 */
  	public boolean isAceCompatible() {
  		return getLanguages().contains(mGfServiceResultGrammar.getName() + SUFFIX_APE);
  	}
  
  
  	/**
  	 * Parses the given text in the given language.
  	 *
  	 * @param text The text.
  	 * @param language The language.
  	 * @return The parse result.
  	 * @throws GfServiceException
  	 */
  	public Set<String> parse(String text, String language) throws GfServiceException {
  		GfServiceResultParse result = mGfService.parse(mCat, text, language, GF_PARSE_LIMIT);
  		return result.getTrees(language);
  	}
  
  
  	public String random() throws GfServiceException {
  		return random(1).iterator().next();
  	}
  
  
  	public List<String> random(int limit) throws GfServiceException {
  		GfServiceResultRandom result = mGfService.random(mCat, limit);
  		return result.getTrees();
  	}
  
  
  	/**
  	 * Serializes the GF wiki entry, given as 3 components:
  	 *   - language (e.g. GeographyEng)
  	 *   - sentence as string (e.g. "Germany is a country .")
  	 *   - set of corresponding trees
  	 *
  	 * The format is:
  	 *
  	 *  lang||text||tree1|tree2|...|treeN
  	 *
  	 * This is more robust, e.g. if the tree cannot be linearized anymore
  	 * because grammar was refactored then we could try to parse the
  	 * sentence. Also the sentence could be shown if the tree
  	 * has multiple variant lins.
  	 */
  	public static String serialize(GfWikiEntry entry) {
  		return GF_SERIALIZATION_JOINER.join(
  				entry.getLanguage(),
  				entry.getText(),
  				GF_TREE_JOINER.join(entry.getTrees().getTrees()));
  	}
  
  
  	/**
  	 * Deserializes a GF wiki entry.
  	 */
  	public static GfWikiEntry deserialize(String serialized) {
  		List<String> splitsAsList = ImmutableList.copyOf(GF_SERIALIZATION_SPLITTER.split(serialized));
  		if (splitsAsList.size() == 1) {
  			// deprecated form, containing just the trees
  			return new GfWikiEntry(new TreeList(GF_TREE_SPLITTER.split(serialized)));
  		} else if (splitsAsList.size() == 3) {
  			Iterable<String> trees = GF_TREE_SPLITTER.split(splitsAsList.get(2));
  			return new GfWikiEntry(
  					splitsAsList.get(0),
  					splitsAsList.get(1),
  					new TreeList(trees));
  		}
  		throw new RuntimeException("Syntax error: " + serialized);
  	}
  
  
  	public Set<String> linearize(String tree, String language) throws GfServiceException {
  		GfServiceResultLinearize result = mGfService.linearize(tree, language);
  		return result.getTexts(language);
  	}
  
  
  	public Map<String, Set<String>> linearize(String tree) throws GfServiceException {
  		GfServiceResultLinearize result = mGfService.linearize(tree, null);
  		return result.getTexts();
  	}
  
  
  	public Set<String> complete(List<String> tokens, String language) throws GfServiceException {
  		return complete(mCat, tokens, language);
  	}
  
  
  	/**
  	 * <p>This method tries to return a set that contains more than one element, i.e.
  	 * if there is only one (unambiguous) completion then "complete" is automatically
  	 * called again. In this case the result set contains multi-token completions.
  	 * There is a limit of 15 tokens to each completion.</p>
  	 *
  	 * @param cat start category for the parser
  	 * @param tokens list of tokens the last of which is to be completed
  	 * @param language language of the input tokens
  	 * @return list of possible completions
  	 * @throws GfServiceException
  	 */
  	public Set<String> complete(String cat, List<String> tokens, String language) throws GfServiceException {
  		// Remove the last argument if this behavior turns out to be confusing
  		// Removed it (was 15), it seemed to be buggy in some cases.
  		GfServiceResultComplete result = mGfService.complete(cat, getCompletionInput(tokens), language, null);
  		return result.getCompletions(language);
  	}
  
  	public String abstrtree(String tree) throws GfServiceException {
  		return mGfService.abstrtree(tree).getDataUri();
  	}
  
  	public String parsetree(String tree, String from) throws GfServiceException {
  		return mGfService.parsetree(tree, from).getDataUri();
  	}
  
  	public String alignment(String tree) throws GfServiceException {
  		return mGfService.alignment(tree).getDataUri();
  	}
  
  
  	public Set<String> getProducers(String cat) {
  		return mGfServiceResultBrowseAll.getProducers(cat);
  	}
  
  
  	public Set<String> getConsumers(String cat) {
  		return mGfServiceResultBrowseAll.getConsumers(cat);
  	}
  
  
  	public String getCategoryName(String cat, String language) {
  		return mGfServiceResultBrowseAll.getCategoryName(cat, language);
  	}
  
  
  	/**
  	 * <p>Returns the {@code k} largest categories in the order of size.
  	 * The size is in terms of the number of producer functions that are
  	 * not consumer functions.</p>
  	 */
  	public List<String> getLargestCategories(int k) {
  		return Ordering.natural().onResultOf(Functions.forMap(mCatToSize)).greatestOf(mCatToSize.keySet(), k);
  	}
  
  	public Multimap<String, String> getTokenToCats(String language) {
  		return langToTokenToCats.get(language);
  	}
  
  	public Map<String, String> getIriToToken(String language) {
  		return langToIriToToken.get(language);
  	}
  
  	public GfParseResult parseGfModule(GfModule gfModule) throws GfServiceException {
  		return mGfStorage.parse(gfModule);
  	}
  
  
  	/**
  	 * Uploads the given GF module to the server.
  	 */
  	public void upload(GfModule module) throws GfServiceException {
  		mGfStorage.upload(mDir, module);
  	}
  
  
  	public Set<String> ls(String extension) throws GfServiceException {
  		GfStorageResultLs result = mGfStorage.ls(mDir, extension);
  		return result.getFilenames();
  	}
  
  
  	public void rm(String path) throws GfServiceException {
  		mGfStorage.rm(mDir, path);
  	}
  
  
  	public int rmGfo() throws GfServiceException {
  		int count = 0;
  		for (String path : ls(EXTENSION_GFO)) {
  			mGfStorage.rm(mDir, path);
  			count++;
  		}
  		return count;
  	}
  
  
  	public String downloadAsString(String filename) throws GfServiceException {
  		return mGfStorage.downloadAsString(mDir, filename);
  	}
  
  
  	/**
  	 * Updates the grammar based on the given GF module, which is either
  	 * a new component of the grammar or which has undergone modifications
  	 * and needs to be reintegrated.
  	 *
  	 * @param gfModule new or modified grammar module
  	 * @return GfStorageResult
  	 * @throws GfServiceException
  	 */
  	public GfStorageResult integrateGfModule(GfModule gfModule) throws GfServiceException {
  		Set<String> languages = getLanguages();
  		GfStorageResult result = null;
  		if (isToplevelModule(gfModule, languages)) {
  			// If the module is a (toplevel) concrete syntax module then
  			// update it in the context of other concrete modules.
  			result = mGfStorage.update(mDir, mCat, OPTIMIZE_PGF, languages, gfModule);
  		} else {
  			// Otherwise just upload it and recompile the existing concrete modules.
  			mGfStorage.upload(mDir, gfModule);
  			result = mGfStorage.update(mDir, mCat, OPTIMIZE_PGF, languages);
  		}
  		if (result != null && result.isSuccess()) {
  			refreshGrammarInfo();
  			refreshLangToTokenToCats();
  		}
  		return result;
  	}
  
  
  	/**
  	 * Recompiles the grammar.
  	 */
  	public GfStorageResult update() throws GfServiceException {
  		Set<String> languages = getLanguages();
  		GfStorageResult result = mGfStorage.update(mDir, mCat, OPTIMIZE_PGF, languages);
  		if (result != null && result.isSuccess()) {
  			refreshGrammarInfo();
  			refreshLangToTokenToCats();
  		}
  		return result;
  	}
  
  
  	public boolean isGrammarEditable() {
  		return ! (mDir == null);
  	}
  
  
  	/**
  	 * True if the module is a concrete syntax module which no other
  	 * module imports. We check if the module name has the form
  	 * {@code GrammarLan}. This covers also modules
  	 * which were added after the wiki was started up. The previous
  	 * technique {@code languages.contains(gfModule.getName())} did not
  	 * cover the new modules.
  	 */
  	private boolean isToplevelModule(GfModule gfModule, Set<String> languages) {
  		String moduleName = gfModule.getName();
  		if (languages.contains(moduleName)) {
  			return true;
  		}
  		if (mGfServiceResultGrammar == null) {
  			return false;
  		}
  		String grammarName = mGfServiceResultGrammar.getName();
  
  		return (
  				moduleName.startsWith(grammarName) &&
  				moduleName.length() >= grammarName.length() + 3 &&
  				Character.isUpperCase(moduleName.charAt(grammarName.length()))
  				||
  				moduleName.startsWith(PREFIX_DISAMB + grammarName) &&
  				moduleName.length() >= PREFIX_DISAMB.length() + grammarName.length() + 3 &&
  				Character.isUpperCase(moduleName.charAt(PREFIX_DISAMB.length() + grammarName.length()))
  				);
  	}
  
  
  	// TODO: we assume that editable directories have a certain form
  	private static String getDir(String str) {
  		Pattern p = Pattern.compile("(/tmp/.+)/.+");
  		Matcher m = p.matcher(str);
  		if (m.matches()) {
  			return m.group(1);
  		}
  		return null;
  	}
  
  
  	private static String getCompletionInput(List<String> tokens) {
  		if (tokens.isEmpty()) {
  			return "";
  		}
  		return GF_TOKEN_JOINER.join(tokens) + GF_TOKEN_SEPARATOR;
  	}
  
  
  	private void refreshGrammarInfo() throws GfServiceException {
  		mGfServiceResultGrammar = mGfService.grammar();
  		mGfServiceResultBrowseAll = mGfService.browseAll();
  	}
  
  
  	/**
  	 * <p>Creates a structure from which you can look up the categories of tokens.</p>
  	 *
  	 * <pre>
  	 * language -> token -> categories
  	 * </pre>
  	 */
  	private void refreshLangToTokenToCats() throws GfServiceException {
  		// Collect together all the consumer functions.
  		// TODO We are not interested in their linearizations, at least for the time begin.
  		Set<String> funsAllConsumers = Sets.newHashSet();
  		Set<String> cats = mGfServiceResultBrowseAll.getCategories();
  		for (String cat : cats) {
  			funsAllConsumers.addAll(getConsumers(cat));
  		}
  
  		int countAllFuns = mGfServiceResultGrammar.getFunctions().size();
  		int countIgnoreFuns = funsAllConsumers.size();
  
  		mLogger.info("All funs: {}, (ignored) consumer funs: {}", countAllFuns, countIgnoreFuns);
  		if (countAllFuns - countIgnoreFuns > LINEARIZE_ALL_QUERY_LIMIT) {
  			mLogger.warn("Refusing to build preditor cache, as there are too many producer-only funs. " +
  					"Increase LINEARIZE_ALL_QUERY_LIMIT if its current value {} is too low.", LINEARIZE_ALL_QUERY_LIMIT);
  			return;
  		}
  
  		langToTokenToCats.clear();
  		mCatToSize.clear();
  		langToIriToToken.clear();
  		// Iterate over all the categories that have producer functions
  		for (String cat : cats) {
  			mCatToSize.put(cat, 0);
  			// For each category look at its producers
  			for (String f : getProducers(cat)) {
  				// If this function is also a consumer, then throw it out
  				if (funsAllConsumers.contains(f)) {
  					continue;
  				}
  				// Increment the counter of producers that are not consumers for this category
  				mCatToSize.put(cat, mCatToSize.get(cat) + 1);
  				// Otherwise get all of its linearizations in all the languages.
  				// This includes all the wordforms and variants, because the linearization
  				// is likely to be a complex record that holds many strings.
  				GfServiceResultLinearizeAll result = mGfService.linearizeAll(f, null);
  				Map<String, List<String>> langToTokens = result.getTexts();
  				// Extract the logical symbol that corresponds to this function.
  				// The logical symbol is present in the Ape-linearization.
  				String logicalSymbol = extractLogicalSymbolFromApe(langToTokens.get(mGfServiceResultGrammar.getName() + SUFFIX_APE));
  				for (Entry<String, List<String>> entry2 : langToTokens.entrySet()) {
  					String lang = entry2.getKey();
  					Multimap<String, String> tokenToCats = langToTokenToCats.get(lang);
  					// If we haven't seen this language before then create a new hash table entry for it
  					if (tokenToCats == null) {
  						tokenToCats = HashMultimap.create();
  						langToTokenToCats.put(lang, tokenToCats);
  					}
  					// Store each linearization together with its category.
  					// The linearization is represented by its "most important" token.
  					for (String lin : entry2.getValue()) {
  						String indexToken = getIndexToken(lin);
  						if (indexToken != null) {
  							tokenToCats.put(indexToken, cat);
  						}
  					}
  
  
  					if (logicalSymbol != null) {
  						Map<String, String> iriToToken = langToIriToToken.get(lang);
  						// If we haven't seen this language before then create a new hash table entry for it
  						if (iriToToken == null) {
  							iriToToken =  Maps.newHashMap();
  							langToIriToToken.put(lang, iriToToken);
  						}
  						for (String lin : entry2.getValue()) {
  							iriToToken.put(logicalSymbol, lin);
  							// TODO: We assume that the dictionary form is always the first.
  							// Of course, this does not always hold.
  							// Unfortunately, LinearizeAll cannot be used to obtain a GF record,
  							// with all the category labels of the strings, but just a list of plain strings.
  							break;
  						}
  					}
  				}
  			}
  		}
  	}
  
  
  	/**
  	 * It does not make sense to index linearizations which contain multiple tokens
  	 * or which are empty strings, as these cannot be matched during (single token)
  	 * lookahead editing. If there are multiple tokens in the given linearization, e.g.
  	 * the + Atlantic_Ocean, des + Atlantischen_Ozeans, Atlandi_Ookean + &+ + il;
  	 * then we return the longest token (picking the last one in case there are several).
  	 * TODO: this is a hack while we're waiting for a cleaner solution.
  	 */
  	private static String getIndexToken(String lin) {
  		int max = 0;
  		String returnTok = null;
  		for (String tok : GF_TOKEN_SPLITTER.omitEmptyStrings().split(lin)) {
  			if (tok.length() >= max) {
  				max = tok.length();
  				returnTok = tok;
  			}
  		}
  		return returnTok;
  	}
  
  
  	/**
  	 * <p>Extracts the logical symbol (which is used by APE as the
  	 * OWL entity IRI) from the Ape-linearization of a function, assuming
  	 * that the function is a lexical function.
  	 * Returns {@code null} in case the extraction fails.</p>
  	 *
  	 * <p>We assume that the Ape linearizations have the form
  	 * {@code The_Hague|pn_sg|The_Hague_PN|neutr}, where the logical symbol
  	 * is always in the same field and is always the same in case there are
  	 * several linearizations.</p>
  	 */
  	private static String extractLogicalSymbolFromApe(List<String> lins) {
  		if (lins == null || lins.isEmpty()) {
  			return null;
  		}
  		int count = 0;
  		for (String field : GF_APE_SPLITTER.split(lins.get(0))) {
  			if (++count == GF_APE_FIELD_LOGICAL_SYMBOL) {
  				return field;
  			}
  		}
  		return null;
  	}
  }