content: package zemberek.corpus;
  
  import java.io.IOException;
  import java.io.PrintWriter;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.stream.Collectors;
  import zemberek.core.logging.Log;
  import zemberek.normalization.TurkishSpellChecker;
  import zemberek.tokenization.TurkishSentenceExtractor;
  import zemberek.tokenization.TurkishTokenizer;
  
  public class SpellingCorpusProducer {
  
    public static void main(String[] args) throws IOException {
      Path rawRoot = Paths.get("/home/ahmetaa/data/text/news-corpus");
      Path sentenceRoot = Paths.get("/home/ahmetaa/data/text/news-sentences");
      Path corpus = Paths.get("/home/ahmetaa/data/text/spelling-corpus");
      extractSentencesFromDir(rawRoot, sentenceRoot);
      extractTokenCorpusFromDir(sentenceRoot, corpus);
    }
  
    private static void extractSentencesFromDir(Path rawRoot, Path sentenceRoot) throws IOException {
      List<Path> paths = Files.walk(rawRoot)
          .filter(s -> s.toFile().getName().endsWith(".corpus")).collect(Collectors.toList());
      SpellingCorpusProducer producer = new SpellingCorpusProducer();
      for (Path in : paths) {
        Path out = sentenceRoot.resolve(in.toFile().getName());
        producer.extractSentences(in, out);
      }
    }
  
    private static void extractTokenCorpusFromDir(Path root, Path out) throws IOException {
      List<Path> paths = Files.walk(root)
          .filter(s -> s.toFile().getName().endsWith(".corpus")).collect(Collectors.toList());
      SpellingCorpusProducer producer = new SpellingCorpusProducer();
      producer.generateCorpus(paths, out);
    }
  
    void extractSentences(Path input, Path output) throws IOException {
  
      TurkishSentenceExtractor extractor = TurkishSentenceExtractor.DEFAULT;
  
      WebCorpus wc = getWebCorpus(input);
  
      Log.info("Processing documents.");
      for (WebDocument doc : wc.getDocuments()) {
        List<String> paragraphs = doc.getLines();
        List<String> sentences = new ArrayList<>(paragraphs.size() * 5);
        for (String paragraph : paragraphs) {
          sentences.addAll(extractor.fromParagraph(paragraph));
        }
        // set new content.
        doc.setContent(sentences);
      }
  
      Log.info("Saving corpus to %s", output);
      wc.save(output, false);
    }
  
    private WebCorpus getWebCorpus(Path input) throws IOException {
      Log.info("Loading %s", input);
      WebCorpus wc = new WebCorpus(input.toFile().getName(), input.toFile().getName());
      wc.addDocuments(WebCorpus.loadDocuments(input));
      return wc;
    }
  
    void generateCorpus(List<Path> sentenceCorpusPaths, Path output) throws IOException {
      TurkishTokenizer lexer = TurkishTokenizer.DEFAULT;
      try (PrintWriter pw = new PrintWriter(output.toFile(), "utf-8")) {
        for (Path input : sentenceCorpusPaths) {
          WebCorpus wc = getWebCorpus(input);
          Log.info("Processing documents.");
          for (WebDocument doc : wc.getDocuments()) {
            List<String> sentences = doc.getLines();
            for (String sentence : sentences) {
              List<String> tokenized = TurkishSpellChecker.tokenizeForSpelling(sentence);
              String tokenSentence = String.join(" ", tokenized);
              pw.println(tokenSentence);
            }
          }
        }
      }
    }
  
  
  }
  