content: package zemberek.tokenization;
  
  import com.google.common.base.Joiner;
  import com.google.common.base.Stopwatch;
  import java.io.BufferedOutputStream;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.OutputStream;
  import java.nio.charset.Charset;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.HashSet;
  import java.util.LinkedHashSet;
  import java.util.List;
  import java.util.Set;
  import java.util.concurrent.TimeUnit;
  import java.util.regex.Pattern;
  import java.util.stream.Collectors;
  import opennlp.tools.cmdline.dictionary.DictionaryBuilderTool;
  import opennlp.tools.dictionary.Dictionary;
  import opennlp.tools.sentdetect.SentenceDetectorFactory;
  import opennlp.tools.sentdetect.SentenceDetectorME;
  import opennlp.tools.sentdetect.SentenceModel;
  import opennlp.tools.sentdetect.SentenceSample;
  import opennlp.tools.sentdetect.SentenceSampleStream;
  import opennlp.tools.util.ObjectStream;
  import opennlp.tools.util.PlainTextByLineStream;
  import opennlp.tools.util.TrainingParameters;
  import zemberek.core.logging.Log;
  import zemberek.core.text.Regexps;
  import zemberek.core.text.TextIO;
  import zemberek.core.text.TokenSequence;
  
  public class SentenceExtractionComparison {
  
    public static void main(String[] args) throws IOException {
  
      trainZemberekSentenceExtractor();
  
      Path test = Paths
          .get("tokenization/src/test/resources/tokenization/Sentence-Boundary-Test.txt");
  
      List<String> testSentences = TextIO.loadLines(test);
  
      Log.info(" \n---------------- Zemberek ------------------\n");
  
      evaluate(testSentences, new TurkishSentenceExtractorAdapter());
  
      Log.info(" \n---------------- OpenNlp ------------------\n");
  /*
      Path openNlpModel = Paths.get("/home/ahmetaa/data/nlp/tr-sent.bin");
      if (!openNlpModel.toFile().exists()) {
        trainOpenNlp();
      }
      SentenceExtractor openNlpAdapter = new OpenNlpAdapter(openNlpModel);
      evaluate(testSentences, openNlpAdapter);
  */
    }
  
    static void trainZemberekSentenceExtractor() throws IOException {
      Path train = Paths
          .get("tokenization/src/test/resources/tokenization/Sentence-Boundary-Train.txt");
      TurkishSentenceExtractor extractor = TurkishSentenceExtractor.Trainer
          .builder(train)
          .iterationCount(30)
          .learningRate(0.03f)
          .shuffleSentences()
          .skipSpaceFrequency(20)
          .lowerCaseFirstLetterFrequency(20)
          .build()
          .train();
      extractor.saveBinary(
          Paths.get("tokenization/src/main/resources/tokenization/sentence-boundary-model.bin"));
    }
  
    private static void evaluate(List<String> referenceSentences, SentenceExtractor detector)
        throws IOException {
  
      // Sanitize input
      referenceSentences = referenceSentences
          .stream()
          .map(s -> s.replaceAll("\\s+", " ").trim())
          .collect(Collectors.toList());
      String joinedSentence = Joiner.on(" ").join(referenceSentences);
  
      // Extract sentences.
      Stopwatch sw = Stopwatch.createStarted();
      List<String> foundSentences = detector.extract(joinedSentence);
      long elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
      Log.info("Extraction Elapsed: %d ms", elapsed);
      Log.info("Speed: %.2f sentences per second", referenceSentences.size() * 1000d / elapsed);
  
      // ---- Evaluation ------
      // separate each boundary token with space in all sentences.
      List<String> separatedSource = new ArrayList<>();
      for (String sentence : referenceSentences) {
        separatedSource.add(sentence
            .replaceAll("[.]", " . ")
            .replaceAll("[?]", " ? ")
            .replaceAll("[!]", " ! ")
            .replaceAll("[…]", " … ")
            .replaceAll("\\s+", " ")
            .trim());
      }
  
      List<String> separatedHypotheses = new ArrayList<>();
      for (String sentence : foundSentences) {
        separatedHypotheses.add(sentence
            .replaceAll("[.]", " . ")
            .replaceAll("[?]", " ? ")
            .replaceAll("[!]", " ! ")
            .replaceAll("[…]", " … ")
            .replaceAll("\\s+", " ")
            .trim());
      }
  
      Set<Integer> refBoundaries = new LinkedHashSet<>();
      int j = 0;
      for (String sentence : separatedSource) {
        TokenSequence seq = new TokenSequence(sentence);
        j = j + seq.size();
        refBoundaries.add(j);
      }
  
      Set<Integer> foundBoundaries = new LinkedHashSet<>();
      int k = 0;
      for (String s : separatedHypotheses) {
        TokenSequence seq = new TokenSequence(s);
        k = k + seq.size();
        foundBoundaries.add(k);
      }
  
      Log.info("Evaluation result for boundary matches.");
      Log.info("Actual sentence count = %d , Found sentence count = %d", referenceSentences.size(),
          foundSentences.size());
      Set<Integer> truePositives = new HashSet<>(refBoundaries);
      truePositives.retainAll(foundBoundaries);
      Set<Integer> falsePositives = new HashSet<>(foundBoundaries);
      falsePositives.removeAll(refBoundaries);
      double precision = truePositives.size() * 1d / (truePositives.size() + falsePositives.size());
      Log.info("Precision       = %.4f", precision);
      double recall = truePositives.size() * 1d / refBoundaries.size();
      Log.info("Recall          = %.4f", recall);
      double f = 2 * precision * recall / (precision + recall);
      Log.info("F               = %.4f", f);
  
      Set<Integer> insertions = new HashSet<>(foundBoundaries);
      insertions.removeAll(refBoundaries);
      Set<Integer> deletions = new HashSet<>(refBoundaries);
      deletions.removeAll(foundBoundaries);
  
      Log.info("NIST error rate = %.4f%%",
          (deletions.size() + insertions.size()) * 100d / refBoundaries.size());
  
      HashSet<String> matchingSentences = new HashSet<>(foundSentences);
      matchingSentences.retainAll(referenceSentences);
      Log.info("Amount of exactly matching sentences = %d in %d (%.4f%%)",
          matchingSentences.size(),
          referenceSentences.size(),
          matchingSentences.size() * 100d / referenceSentences.size());
    }
  
    public static void findCandidateTrainingSentences() throws IOException {
      Pattern p = Pattern.compile("[ ][a-zA-ZişüğıöçÜĞÖİŞÇ0-9,.'\\-%]+[.!?]+[ ]");
      List<String> input = Files
          .readAllLines(Paths.get("/media/depo/data/aaa/nlp/sentences/k/2017-01-04"));
      TurkishSentenceExtractor extractor = TurkishSentenceExtractor.DEFAULT;
      List<String> extracted = extractor.fromParagraphs(input);
      List<String> foo = extracted
          .stream()
          .filter(s -> Regexps.matchesAny(p, s) && s.length() < 150)
          .collect(Collectors.toList());
  
      Files.write(Paths.get("/media/depo/data/aaa/nlp/sentences/candidate"), foo);
    }
  
    public static void trainOpenNlp() throws IOException {
      Charset charset = Charset.forName("UTF-8");
      ObjectStream<String> lineStream =
          new PlainTextByLineStream(() -> new FileInputStream(
              Paths.get("tokenization/src/test/resources/tokenization/Sentence-Boundary-Train.txt").toFile()),
              charset);
      ObjectStream<SentenceSample> sampleStream = new SentenceSampleStream(lineStream);
  
      File abbrvFile = new File("/media/depo/data/aaa/nlp/abbreviations");
      File abbrvOut = new File("/media/depo/data/aaa/nlp/abbreviations.dict");
  
      SentenceModel model;
      new DictionaryBuilderTool().run(
          new String[]{
              "-inputFile",
              abbrvFile.getAbsolutePath(),
              "-outputFile",
              abbrvOut.getAbsolutePath(),
              "-encoding",
              "utf-8"});
      opennlp.tools.dictionary.Dictionary dictionary = new Dictionary(new FileInputStream(abbrvOut));
      SentenceDetectorFactory factory =
          new SentenceDetectorFactory("tr", true, dictionary, ".?!".toCharArray());
  
      model = SentenceDetectorME
          .train("tr", sampleStream, factory, TrainingParameters.defaultParams());
  
      OutputStream modelOut = new BufferedOutputStream(
          new FileOutputStream(new File("/media/depo/data/aaa/nlp/sentence.bin")));
      model.serialize(modelOut);
    }
  
    interface SentenceExtractor {
  
      List<String> extract(String paragraph);
  
      List<String> extract(List<String> paragraphs);
    }
  
    static class TurkishSentenceExtractorAdapter implements SentenceExtractor {
  
      TurkishSentenceExtractor sentenceDetector = TurkishSentenceExtractor.DEFAULT;
  
      @Override
      public List<String> extract(String paragraph) {
        return sentenceDetector.fromParagraph(paragraph);
      }
  
      @Override
      public List<String> extract(List<String> paragraphs) {
        return sentenceDetector.fromParagraphs(paragraphs);
      }
    }
  
    static class OpenNlpAdapter implements SentenceExtractor {
  
      SentenceDetectorME sentenceDetector;
  
      OpenNlpAdapter(Path modelPath) throws IOException {
        SentenceModel model = new SentenceModel(modelPath.toFile());
        this.sentenceDetector = new SentenceDetectorME(model);
      }
  
      @Override
      public List<String> extract(String paragraph) {
        return Arrays.asList(sentenceDetector.sentDetect(paragraph));
      }
  
      @Override
      public List<String> extract(List<String> paragraphs) {
        List<String> result = new ArrayList<>();
        for (String paragraph : paragraphs) {
          result.addAll(extract(paragraph));
        }
        return result;
      }
    }
  
  
  }
  