content: package zemberek.morphology.old_ambiguity;
  
  import com.google.common.base.Joiner;
  import com.google.common.base.Splitter;
  import com.google.common.base.Stopwatch;
  import com.google.common.collect.Maps;
  import java.io.File;
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Locale;
  import java.util.Map;
  import java.util.concurrent.TimeUnit;
  import zemberek.core.collections.Histogram;
  import zemberek.core.io.LineIterator;
  import zemberek.core.io.SimpleTextReader;
  import zemberek.core.text.TextUtil;
  import zemberek.morphology.TurkishMorphology;
  import zemberek.morphology.analysis.WordAnalysis;
  import zemberek.tokenization.TurkishTokenizer;
  
  /**
   * A class for calculating basic morphological ambiguity statistics
   *
   * @author mdakin@gmail.com
   */
  public class AmbiguityStats {
  
    private static final Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();
    TurkishMorphology parser;
    TurkishTokenizer lexer = TurkishTokenizer.DEFAULT;
  
    public AmbiguityStats() throws IOException {
      parser = TurkishMorphology.createWithDefaults();
    }
  
    public static void main(String[] args) throws IOException {
      AmbiguityStats parser = new AmbiguityStats();
      Stopwatch w = Stopwatch.createStarted();
      //parser.ambiguousWordStats("File with turkish sentences");
      //parser.ambiguousGroupStats("File with turkish sentences");
      System.out.println(w.elapsed(TimeUnit.MILLISECONDS) + " ms.");
    }
  
    public List<String> readAll(String filename) throws IOException {
      List<String> lines = new ArrayList<>();
      File file = new File(filename);
      LineIterator it = SimpleTextReader.trimmingUTF8Reader(file).getLineIterator();
      while (it.hasNext()) {
        String quotesHyphensNormalzied = TextUtil.normalizeQuotesHyphens(it.next());
        lines.add(Joiner.on(" ").join(lexer.tokenizeToStrings(quotesHyphensNormalzied)));
      }
      return lines;
    }
  
    public String pp(String percent) {
      return " (%" + percent + ")";
    }
  
    public String percentStr(int p1, int p2) {
      return String.format(Locale.ENGLISH,"%.2f", pct(p1, p2));
    }
  
    public String percentStr3(int p1, int p2) {
      return String.format(Locale.ENGLISH,"%.3f", pct(p1, p2));
    }
  
    public double pct(int p1, int p2) {
      return (double) p1 / p2 * 100;
    }
  
    public void ambiguousGroupStats(String filename) throws IOException {
      List<String> lines = readAll(filename);
      Histogram<String> uniques = new Histogram<>(1000000);
      Map<String, Histogram<String>> ambiguityGroups = Maps.newHashMap();
      int total = 0;
      for (String line : lines) {
        for (String s : splitter.split(line)) {
          WordAnalysis results = parser.analyze(s);
          if (++total % 50000 == 0) {
            System.out.println("Processed: " + total);
          }
          if (results.analysisCount() > 1) {
            String key = generateKeyFromParse(results);
            uniques.add(key);
            Histogram<String> members = ambiguityGroups.get(key);
            if (members == null) {
              members = new Histogram<>();
              ambiguityGroups.put(key, members);
            }
            members.add(s);
          }
        }
      }
      System.out.println("Total: " + total);
      Stats st = new Stats(0.1);
      st.allCounts = (int) uniques.totalCount();
      st.allUniques = uniques.size();
      for (String s : uniques.getSortedList()) {
        int count = uniques.getCount(s);
        if (st.overCutoff(count)) {
          String p1 = percentStr(count, st.allCounts);
          st.significantCounts += count;
          st.significantUniques++;
          System.out.println(s + " : " + count + "    " + pp(p1));
          Histogram<String> members = ambiguityGroups.get(s);
          for (String member : members.getSortedList()) {
            int memberCount = members.getCount(member);
            if (pct(memberCount, count) > 0.1) {
              System.out.println(member + " : " + members.getCount(member));
            }
          }
          System.out.println();
        }
      }
      st.dump();
    }
  
    private String generateKeyFromParse(WordAnalysis results) {
      StringBuilder key = new StringBuilder();
      for (int i = 0; i < results.analysisCount(); i++) {
        key.append(results.getAnalysisResults().get(i).formatMorphemesLexical());
        if (i < results.analysisCount() - 1) {
          key.append("\n");
        }
      }
      return key.toString();
    }
  
    public void ambiguousWordStats(String filename) throws IOException {
      List<String> lines = readAll(filename);
      Histogram<String> uniques = new Histogram<>(1000000);
      int total = 0;
      Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();
      for (String line : lines) {
        for (String s : splitter.split(line)) {
          WordAnalysis results = parser.analyze(s);
          total++;
          if (total % 50000 == 0) {
            System.out.println("Processed: " + total);
          }
          if (results.analysisCount() > 1) {
            uniques.add(s);
          }
        }
      }
      System.out.println("Total: " + total);
      Stats st = new Stats(0.002);
      st.allCounts = (int) uniques.totalCount();
      st.allUniques = uniques.size();
      for (String s : uniques.getSortedList()) {
        int count = uniques.getCount(s);
        if (st.overCutoff(count)) {
          String p1 = percentStr3(count, st.allCounts);
          st.significantCounts += count;
          st.significantUniques++;
          System.out.println(s + " : " + count + "    " + pp(p1));
        }
      }
      st.dump();
    }
  
    public void noParse(String... filename) throws IOException {
      Histogram<String> uniques = new Histogram<>(1000000);
      int total = 0;
      for (String file : filename) {
        List<String> lines = readAll(file);
        Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();
        for (String line : lines) {
          for (String s : splitter.split(line)) {
            WordAnalysis results = parser.analyze(s);
            total++;
            if (total % 50000 == 0) {
              System.out.println("Processed: " + total);
            }
            if (results.analysisCount() == 0) {
              uniques.add(s);
            }
          }
        }
  
        System.out.println("Total: " + total);
      }
      Stats st = new Stats(0.0002);
      st.allCounts = (int) uniques.totalCount();
      st.allUniques = uniques.size();
      for (String s : uniques.getSortedList()) {
        int count = uniques.getCount(s);
        if (count > 5) {
          st.significantCounts += count;
          st.significantUniques++;
          System.out.println(s + " : " + count);
        }
      }
      st.dump();
    }
  
    class Stats {
  
      int significantCounts = 0;
      int significantUniques = 0;
      int allCounts = 0;
      int allUniques = 0;
      double cutoff;
  
      // cutoff: percentage 10 = %10 etc.
      Stats(double cutoff) {
        this.cutoff = cutoff;
      }
  
      boolean overCutoff(int x) {
        return ((double) x / allCounts * 100) > cutoff;
      }
  
      void dump() {
        String pCounts = percentStr(significantCounts, allCounts);
        String pUniques = percentStr(significantUniques, allUniques);
        System.out.println();
        System.out.println(
            "Significants/All(Counts): " + significantCounts + "/" + allCounts + pp(pCounts));
        System.out.println(
            "Significants/All(Uniques): " + significantUniques + "/" + allUniques + pp(pUniques));
        System.out.println();
      }
    }
  }
  