content: /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.solr.spelling;
  
  import java.io.IOException;
  import java.io.StringReader;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import org.apache.lucene.analysis.Token;
  import org.apache.lucene.analysis.TokenStream;
  import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
  import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
  import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
  import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
  import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
  
  
  /**
   * Converts the query string to a Collection of Lucene tokens using a regular expression.
   * Boolean operators AND and OR are skipped.
   *
   * @since solr 1.3
   **/
  public class SpellingQueryConverter extends QueryConverter  {
  
    /*
    * The following builds up a regular expression that matches productions
    * of the syntax for NMTOKEN as per the W3C XML Recommendation - with one
    * important exception (see below).
    *
    * http://www.w3.org/TR/2008/REC-xml-20081126/ - version used as reference
    *
    * http://www.w3.org/TR/REC-xml/#NT-Nmtoken
    *
    * An NMTOKEN is a series of one or more NAMECHAR characters, which is an
    * extension of the NAMESTARTCHAR character class.
    *
    * The EXCEPTION referred to above concerns the colon, which is legal in an
    * NMTOKEN, but cannot currently be used as a valid field name within Solr,
    * as it is used to delimit the field name from the query string.
    */
  
    final static String[] NAMESTARTCHAR_PARTS = {
            "A-Z_a-z", "\\xc0-\\xd6", "\\xd8-\\xf6", "\\xf8-\\u02ff",
            "\\u0370-\\u037d", "\\u037f-\\u1fff",
            "\\u200c-\\u200d", "\\u2070-\\u218f",
            "\\u2c00-\\u2fef", "\\u2001-\\ud7ff",
            "\\uf900-\\ufdcf", "\\ufdf0-\\ufffd"
    };
    final static String[] ADDITIONAL_NAMECHAR_PARTS = {
            "\\-.0-9\\xb7", "\\u0300-\\u036f", "\\u203f-\\u2040"
    };
    final static String SURROGATE_PAIR = "\\p{Cs}{2}";
    final static String NMTOKEN;
  
    static {
      StringBuilder sb = new StringBuilder();
      for (String part : NAMESTARTCHAR_PARTS)
        sb.append(part);
      for (String part : ADDITIONAL_NAMECHAR_PARTS)
        sb.append(part);
      NMTOKEN = "([" + sb.toString() + "]|" + SURROGATE_PAIR + ")+";
    }
  
    final static String PATTERN = "(?:(?!(" + NMTOKEN + ":|\\d+)))[\\p{L}_\\-0-9]+";
    // previous version: Pattern.compile("(?:(?!(\\w+:|\\d+)))\\w+");
    protected Pattern QUERY_REGEX = Pattern.compile(PATTERN);
  
  
    /**
     * Converts the original query string to a collection of Lucene Tokens.
     * @param original the original query string
     * @return a Collection of Lucene Tokens
     */
    @Override
    public Collection<Token> convert(String original) {
      if (original == null) { // this can happen with q.alt = and no query
        return Collections.emptyList();
      }
      Collection<Token> result = new ArrayList<Token>();
      //TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream
      Matcher matcher = QUERY_REGEX.matcher(original);
      TokenStream stream;
      while (matcher.find()) {
        String word = matcher.group(0);
        if (word.equals("AND") == false && word.equals("OR") == false) {
          try {
            stream = analyzer.reusableTokenStream("", new StringReader(word));
            // TODO: support custom attributes
            CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
            FlagsAttribute flagsAtt = stream.addAttribute(FlagsAttribute.class);
            TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);
            PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);
            PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
            stream.reset();
            while (stream.incrementToken()) {
              Token token = new Token();
              token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
              token.setStartOffset(matcher.start());
              token.setEndOffset(matcher.end());
              token.setFlags(flagsAtt.getFlags());
              token.setType(typeAtt.type());
              token.setPayload(payloadAtt.getPayload());
              token.setPositionIncrement(posIncAtt.getPositionIncrement());
              result.add(token);
            }
            stream.end();
            stream.close();
          } catch (IOException e) {
          }
        }
      }
      return result;
    }
  
  }
  
  