content: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.cassandra.index.sasi.analyzer;
  
  import java.io.InputStream;
  import java.nio.ByteBuffer;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Locale;
  
  import org.junit.Test;
  
  import org.apache.cassandra.serializers.UTF8Serializer;
  
  import static org.junit.Assert.assertEquals;
  
  public class StandardAnalyzerTest
  {
      @Test
      public void testTokenizationAscii() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/apache_license_header.txt");
  
          StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder()
                  .maxTokenLength(5).build();
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(options);
  
          List<ByteBuffer> tokens = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(67, tokens.size());
      }
  
      @Test
      public void testTokenizationLoremIpsum() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/lorem_ipsum.txt");
  
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
  
          List<ByteBuffer> tokens = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(62, tokens.size());
  
      }
  
      @Test
      public void testTokenizationJaJp1() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/ja_jp_1.txt");
  
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
  
          tokenizer.reset(is);
          List<ByteBuffer> tokens = new ArrayList<>();
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(210, tokens.size());
      }
  
      @Test
      public void testTokenizationJaJp2() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/ja_jp_2.txt");
  
          StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
                  .ignoreStopTerms(true).alwaysLowerCaseTerms(true).build();
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(options);
  
          tokenizer.reset(is);
          List<ByteBuffer> tokens = new ArrayList<>();
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(57, tokens.size());
      }
  
      @Test
      public void testTokenizationRuRu1() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/ru_ru_1.txt");
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
  
          List<ByteBuffer> tokens = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(456, tokens.size());
      }
  
      @Test
      public void testTokenizationZnTw1() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/zn_tw_1.txt");
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
  
          List<ByteBuffer> tokens = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(963, tokens.size());
      }
  
      @Test
      public void testTokenizationAdventuresOfHuckFinn() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/adventures_of_huckleberry_finn_mark_twain.txt");
  
          StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
                  .ignoreStopTerms(true).useLocale(Locale.ENGLISH)
                  .alwaysLowerCaseTerms(true).build();
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(options);
  
          List<ByteBuffer> tokens = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(37739, tokens.size());
      }
  
      @Test
      public void testSkipStopWordBeforeStemmingFrench() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                 .getResourceAsStream("tokenization/french_skip_stop_words_before_stemming.txt");
  
          StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
                  .ignoreStopTerms(true).useLocale(Locale.FRENCH)
                  .alwaysLowerCaseTerms(true).build();
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(options);
  
          List<ByteBuffer> tokens = new ArrayList<>();
          List<String> words = new ArrayList<>();
          tokenizer.reset(is);
          while (tokenizer.hasNext())
          {
              final ByteBuffer nextToken = tokenizer.next();
              tokens.add(nextToken);
              words.add(UTF8Serializer.instance.deserialize(nextToken.duplicate()));
          }
  
          assertEquals(4, tokens.size());
          assertEquals("dans", words.get(0));
          assertEquals("plui", words.get(1));
          assertEquals("chanson", words.get(2));
          assertEquals("connu", words.get(3));
      }
  
      @Test
      public void tokenizeDomainNamesAndUrls() throws Exception
      {
          InputStream is = StandardAnalyzerTest.class.getClassLoader()
                  .getResourceAsStream("tokenization/top_visited_domains.txt");
  
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
          tokenizer.reset(is);
  
          List<ByteBuffer> tokens = new ArrayList<>();
          while (tokenizer.hasNext())
              tokens.add(tokenizer.next());
  
          assertEquals(15, tokens.size());
      }
  
      @Test
      public void testReuseAndResetTokenizerInstance() throws Exception
      {
          List<ByteBuffer> bbToTokenize = new ArrayList<>();
          bbToTokenize.add(ByteBuffer.wrap("Nip it in the bud".getBytes()));
          bbToTokenize.add(ByteBuffer.wrap("I couldn’t care less".getBytes()));
          bbToTokenize.add(ByteBuffer.wrap("One and the same".getBytes()));
          bbToTokenize.add(ByteBuffer.wrap("The squeaky wheel gets the grease.".getBytes()));
          bbToTokenize.add(ByteBuffer.wrap("The pen is mightier than the sword.".getBytes()));
  
          StandardAnalyzer tokenizer = new StandardAnalyzer();
          tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
  
          List<ByteBuffer> tokens = new ArrayList<>();
          for (ByteBuffer bb : bbToTokenize)
          {
              tokenizer.reset(bb);
              while (tokenizer.hasNext())
                  tokens.add(tokenizer.next());
          }
          assertEquals(10, tokens.size());
      }
  }
  