content: /**
   * Copyright 2000-2006 DFKI GmbH.
   * All Rights Reserved.  Use is subject to license terms.
   * <p/>
   * This file is part of MARY TTS.
   * <p/>
   * MARY TTS is free software: you can redistribute it and/or modify
   * it under the terms of the GNU Lesser General Public License as published by
   * the Free Software Foundation, version 3 of the License.
   * <p/>
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU Lesser General Public License for more details.
   * <p/>
   * You should have received a copy of the GNU Lesser General Public License
   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
   */
  package marytts.modules;
  
  import android.util.Log;
  
  import java.text.CharacterIterator;
  import java.util.Locale;
  
  import de.dfki.lt.tools.tokenizer.JTok;
  import de.dfki.lt.tools.tokenizer.annotate.AnnotatedString;
  import de.dfki.lt.tools.tokenizer.annotate.FastAnnotatedString;
  import marytts.datatypes.MaryData;
  import marytts.datatypes.MaryDataType;
  import marytts.datatypes.MaryXML;
  import marytts.server.Mary;
  import marytts.util.dom.DomUtils;
  import marytts.util.dom.MaryDomUtils;
  import mf.org.w3c.dom.Document;
  import mf.org.w3c.dom.Element;
  import mf.org.w3c.dom.Text;
  import mf.org.w3c.dom.traversal.DocumentTraversal;
  import mf.org.w3c.dom.traversal.NodeFilter;
  import mf.org.w3c.dom.traversal.NodeIterator;
  
  /**
   * @author Marc Schr&ouml;der
   */
  public class JTokeniser extends InternalModule {
      public static final int TOKEN_MAXLENGTH = 100;
  
      private JTok jtok;
      private String jtokLocale;
  
      public JTokeniser() {
          this((Locale) null);
      }
  
      public JTokeniser(String locale) {
          super("JTokeniser", MaryDataType.RAWMARYXML, MaryDataType.TOKENS, new Locale(locale));
      }
  
      public JTokeniser(Locale locale) {
          this(MaryDataType.RAWMARYXML, MaryDataType.TOKENS, locale);
      }
  
      public JTokeniser(MaryDataType inputType, MaryDataType outputType, Locale locale) {
          super("JTokeniser", inputType, outputType, locale);
          // Which language to use in the Tokenizer?
          if (locale == null) {
              // if locale == null, use English tokeniser as default/fallback tokeniser 
              jtokLocale = "en";
          } else {
              jtokLocale = locale.getLanguage();
          }
      }
  
      /**
       * Set the tokenizer language to be different from the Locale of the module.
       * This can be useful when reusing another language's tokenizer data.
       *
       * @param languageCode the language-code to use, as a two-character string such as "de" or "en".
       */
      protected void setTokenizerLanguage(String languageCode) {
          jtokLocale = languageCode;
      }
  
      @Override
      public void startup() throws Exception {
          super.startup();
  //         Properties jtokProperties = new Properties();
  //         jtokProperties.setProperty("languages", jtokLocale);
  //         jtokProperties.setProperty(jtokLocale, "jtok/"+jtokLocale);
          jtok = new JTok();
      }
  
      @Override
      public MaryData process(MaryData d)
              throws Exception {
          Document doc = d.getDocument();
  
          // The challenge in this module is that the tokeniser needs a plain text
          // version of this XML document, but we need to feed its output
          // back into the XML document.
          // Solution strategy: Remember the alignment of text with XML as an
          // annotated string. Each stretch of characters is annotated with the
          // DOM Text object to which it belonged. 
          NodeIterator ni = ((DocumentTraversal) doc).createNodeIterator(
                  doc, NodeFilter.SHOW_TEXT, null, false);
          Text textNode;
          StringBuilder inputText = new StringBuilder();
          // Keep this loop in sync with the second loop, below:
          while ((textNode = (Text) ni.nextNode()) != null) {
              String text = textNode.getData().trim();
              //text = text.replaceAll("\\.", " \\.");
              if (text.length() == 0) continue;
              // Insert a space character between non-punctuation characters:
              if (inputText.length() > 0 &&
                      !Character.isWhitespace(inputText.charAt(inputText.length() - 1)) &&
                      Character.isLetterOrDigit(text.charAt(0))) {
                  inputText.append(" ");
              }
              inputText.append(text);
          }
          FastAnnotatedString maryText = new FastAnnotatedString(inputText.toString());
          // And now go through the TEXT nodes a second time and annotate
          // the string.
          ni = ((DocumentTraversal) doc).createNodeIterator(
                  doc, NodeFilter.SHOW_TEXT, null, false);
          int pos = 0;
          // Keep this loop in sync with the first loop, above:
          while ((textNode = (Text) ni.nextNode()) != null) {
              String text = textNode.getData().trim();
              //text = text.replaceAll("\\.", " \\.");
              int len = text.length();
              if (len == 0) continue;
              // Skip the space character between non-punctuation characters:
              if (pos > 0 &&
                      !Character.isWhitespace(inputText.charAt(pos - 1)) &&
                      Character.isLetterOrDigit(text.charAt(0))) {
                  pos++;
              }
              maryText.annotate("MARYXML", textNode, pos, pos + len);
              pos += len;
          }
          // Now maryText is the input text annotated with the Text nodes in the
          // MaryXML document.
          // Tokenise:
          AnnotatedString tokenisedText;
          tokenisedText = jtok.tokenize(inputText.toString(), jtokLocale);
          //System.err.println("maryText: `" + maryText.toString() + "'"); 
          //System.err.println("tokenisedText: `" + tokenisedText.toString() + "'"); 
          //assert tokenisedText.toString().equals(maryText.toString());
          // And now merge the output back into the MaryXML document.
          Element firstTokenInSentence = null;
          Element firstTokenInParagraph = null;
          Element previousToken = null;
          Text currentTextNode = null;
          char c = tokenisedText.setIndex(0);
          maryText.setIndex(0);
          while (c != CharacterIterator.DONE) {
              int tokenStart = tokenisedText.getRunStart(JTok.CLASS_ANNO);
              int tokenEnd = tokenisedText.getRunLimit(JTok.CLASS_ANNO);
              // check if c belongs to a token
              if (null != tokenisedText.getAnnotation(JTok.CLASS_ANNO)) {
                  // We don't care about the actual annotation, only that there is one.
                  // Where to insert the token:
                  maryText.setIndex(tokenStart);
                  Text tn = (Text) maryText.getAnnotation("MARYXML");
                  assert tn != null;
                  // Only create new token tag if there was none around the corresponding
                  // text node in the input:
                  Element t = null;
                  if (DomUtils.hasAncestor(tn, MaryXML.TOKEN)) {
                      t = (Element) DomUtils.getAncestor(tn, MaryXML.TOKEN);
                  } else { // need to create a new <t> element:
                      String token = tokenisedText.substring(tokenStart, tokenEnd);
                      t = MaryXML.createElement(doc, MaryXML.TOKEN);
                      MaryDomUtils.setTokenText(t, token);
                      // Insert the new token element
                      tn.getParentNode().insertBefore(t, tn);
                  }
                  // If we have by now moved past currentTextNode, delete it
                  // (unless it was already inside a t tag):
                  if (currentTextNode != null && currentTextNode != tn &&
                          !DomUtils.hasAncestor(currentTextNode, MaryXML.TOKEN)) {
                      currentTextNode.getParentNode().removeChild(currentTextNode);
                  }
                  currentTextNode = tn;
                  // Is this token the first in a new sentence or paragraph?
                  if (null != tokenisedText.getAnnotation(JTok.BORDER_ANNO)) {
                      // Yes, this is the first token of a new sentence
                      // Need to enclose previous sentence with an <s> tag?
                      if (firstTokenInSentence != null) {
                          assert previousToken != null;
                          if (!DomUtils.hasAncestor(firstTokenInSentence, MaryXML.SENTENCE) &&
                                  !DomUtils.hasAncestor(previousToken, MaryXML.SENTENCE)) {
                              Element firstPara = (Element) DomUtils.getAncestor(firstTokenInSentence, MaryXML.PARAGRAPH);
                              Element lastPara = (Element) DomUtils.getAncestor(previousToken, MaryXML.PARAGRAPH);
                              if (firstPara == null && lastPara == null || firstPara.equals(lastPara)) {
                                  // Either both tokens have no surrounding paragraph, or both have
                                  // the same.
                                  encloseWithSentence(firstTokenInSentence, previousToken);
                              }
                          }
                      }
                      firstTokenInSentence = null;
                      // Is it even a paragraph start?
                      if (tokenisedText.getAnnotation(JTok.BORDER_ANNO) ==
                              JTok.P_BORDER) {
                          // Need to enclose previous paragraph with a <p> tag?
                          if (firstTokenInParagraph != null) {
                              assert previousToken != null;
                              if (!DomUtils.hasAncestor(firstTokenInParagraph, MaryXML.PARAGRAPH) &&
                                      !DomUtils.hasAncestor(previousToken, MaryXML.PARAGRAPH)) {
                                  DomUtils.encloseNodesWithNewElement(
                                          DomUtils.getAncestor(firstTokenInParagraph, MaryXML.SENTENCE),
                                          DomUtils.getAncestor(previousToken, MaryXML.SENTENCE),
                                          MaryXML.PARAGRAPH);
                              }
                          }
                          firstTokenInParagraph = null;
                      }
                  }
                  previousToken = t;
                  if (firstTokenInSentence == null) firstTokenInSentence = t;
                  if (firstTokenInParagraph == null) firstTokenInParagraph = t;
              }
              c = tokenisedText.setIndex(tokenEnd);
              maryText.setIndex(tokenEnd);
          }
          // Remove the last old text node
          if (currentTextNode != null // should be
                  && !DomUtils.hasAncestor(currentTextNode, MaryXML.TOKEN))
              currentTextNode.getParentNode().removeChild(currentTextNode);
          // Insert the last <s> element
          if (firstTokenInSentence != null) {
              assert previousToken != null;
              if (!DomUtils.hasAncestor(firstTokenInSentence, MaryXML.SENTENCE) &&
                      !DomUtils.hasAncestor(previousToken, MaryXML.SENTENCE)) {
                  Element firstPara = (Element) DomUtils.getAncestor(firstTokenInSentence, MaryXML.PARAGRAPH);
                  Element lastPara = (Element) DomUtils.getAncestor(previousToken, MaryXML.PARAGRAPH);
                  if (firstPara == null && lastPara == null || firstPara.equals(lastPara)) {
                      // Either both tokens have no surrounding paragraph, or both have
                      // the same.
                      encloseWithSentence(firstTokenInSentence, previousToken);
                  }
              }
          }
          // Insert the last <p> element
          if (firstTokenInParagraph != null) {
              assert previousToken != null;
              if (!DomUtils.hasAncestor(firstTokenInParagraph, MaryXML.PARAGRAPH) &&
                      !DomUtils.hasAncestor(previousToken, MaryXML.PARAGRAPH)) {
                  DomUtils.encloseNodesWithNewElement(
                          DomUtils.getAncestor(firstTokenInParagraph, MaryXML.SENTENCE),
                          DomUtils.getAncestor(previousToken, MaryXML.SENTENCE),
                          MaryXML.PARAGRAPH);
              }
          }
  
          // And finally, we make sure for every token that it is not longer than TOKEN_MAXLENGTH:
          NodeIterator tIt = DomUtils.createNodeIterator(doc.getDocumentElement(), MaryXML.TOKEN);
          Element t = null;
          while ((t = (Element) tIt.nextNode()) != null) {
              String tokenText = MaryDomUtils.tokenText(t);
              if (tokenText.length() > TOKEN_MAXLENGTH) {
                  String cutTT = tokenText.substring(0, TOKEN_MAXLENGTH);
                  Log.i(Mary.LOG, "Cutting exceedingly long input token (length " + tokenText.length() + " ) to length " + TOKEN_MAXLENGTH + ":\n"
                          + "before: " + tokenText
                          + "\nafter: " + cutTT);
                  MaryDomUtils.setTokenText(t, cutTT);
              }
          }
  
          MaryData result = new MaryData(outputType(), d.getLocale());
          result.setDocument(doc);
          return result;
      }
  
      /**
       * @param firstTokenInSentence
       * @param lastTokenInSentence
       */
      private void encloseWithSentence(Element firstTokenInSentence, Element lastTokenInSentence) {
          // Allow for sentences to start with a <boundary/> element:
          Element encloseFromHere = firstTokenInSentence;
          Element maybeBoundary = DomUtils.getPreviousSiblingElement(firstTokenInSentence);
          if (maybeBoundary != null && maybeBoundary.getTagName().equals(MaryXML.BOUNDARY)) {
              encloseFromHere = maybeBoundary;
          }
          Element encloseToHere = lastTokenInSentence;
          maybeBoundary = DomUtils.getNextSiblingElement(lastTokenInSentence);
          if (maybeBoundary != null && maybeBoundary.getTagName().equals(MaryXML.BOUNDARY)) {
              encloseToHere = maybeBoundary;
          }
          DomUtils.encloseNodesWithNewElement(encloseFromHere, encloseToHere, MaryXML.SENTENCE);
      }
  
  
  }
  
  