content: package org.apache.lucene.analysis.query;
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.index.TermEnum;
  import org.apache.lucene.analysis.Analyzer;
  import org.apache.lucene.analysis.TokenStream;
  import org.apache.lucene.analysis.StopFilter;
  import org.apache.lucene.util.StringHelper;
  import org.apache.lucene.util.Version;
  
  import java.io.IOException;
  import java.io.Reader;
  import java.util.*;
  
  /**
   * An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection
   * which prevents very common words from being passed into queries. 
   * <p>
   * For very large indexes the cost
   * of reading TermDocs for a very common word can be  high. This analyzer was created after experience with
   * a 38 million doc index which had a term in around 50% of docs and was causing TermQueries for 
   * this term to take 2 seconds.
   * </p>
   * <p>
   * Use the various "addStopWords" methods in this class to automate the identification and addition of 
   * stop words found in an already existing index.
   * </p>
   */
  public final class QueryAutoStopWordAnalyzer extends Analyzer {
  
    private final Analyzer delegate;
    private final Map<String, Set<String>> stopWordsPerField = new HashMap<String, Set<String>>();
    //The default maximum percentage (40%) of index documents which
    //can contain a term, after which the term is considered to be a stop word.
    public static final float defaultMaxDocFreqPercent = 0.4f;
    private final Version matchVersion;
  
    /**
     * Initializes this analyzer with the Analyzer object that actually produces the tokens
     *
     * @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
     * @deprecated Stopwords should be calculated at instantiation using one of the other constructors
     */
    @Deprecated
    public QueryAutoStopWordAnalyzer(Version matchVersion, Analyzer delegate) {
      this.delegate = delegate;
      this.matchVersion = matchVersion;
    }
  
     /**
     * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
     * indexed fields from terms with a document frequency percentage greater than
     * {@link #defaultMaxDocFreqPercent}
     *
     * @param matchVersion Version to be used in {@link StopFilter}
     * @param delegate Analyzer whose TokenStream will be filtered
     * @param indexReader IndexReader to identify the stopwords from
     * @throws IOException Can be thrown while reading from the IndexReader
     */
    public QueryAutoStopWordAnalyzer(
        Version matchVersion,
        Analyzer delegate,
        IndexReader indexReader) throws IOException {
      this(matchVersion, delegate, indexReader, defaultMaxDocFreqPercent);
    }
  
    /**
     * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
     * indexed fields from terms with a document frequency greater than the given
     * maxDocFreq
     *
     * @param matchVersion Version to be used in {@link StopFilter}
     * @param delegate Analyzer whose TokenStream will be filtered
     * @param indexReader IndexReader to identify the stopwords from
     * @param maxDocFreq Document frequency terms should be above in order to be stopwords
     * @throws IOException Can be thrown while reading from the IndexReader
     */
    public QueryAutoStopWordAnalyzer(
        Version matchVersion,
        Analyzer delegate,
        IndexReader indexReader,
        int maxDocFreq) throws IOException {
      this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxDocFreq);
    }
  
    /**
     * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
     * indexed fields from terms with a document frequency percentage greater than
     * the given maxPercentDocs
     *
     * @param matchVersion Version to be used in {@link StopFilter}
     * @param delegate Analyzer whose TokenStream will be filtered
     * @param indexReader IndexReader to identify the stopwords from
     * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
     *                      contain a term, after which the word is considered to be a stop word
     * @throws IOException Can be thrown while reading from the IndexReader
     */
    public QueryAutoStopWordAnalyzer(
        Version matchVersion,
        Analyzer delegate,
        IndexReader indexReader,
        float maxPercentDocs) throws IOException {
      this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxPercentDocs);
    }
  
    /**
     * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for the
     * given selection of fields from terms with a document frequency percentage
     * greater than the given maxPercentDocs
     *
     * @param matchVersion Version to be used in {@link StopFilter}
     * @param delegate Analyzer whose TokenStream will be filtered
     * @param indexReader IndexReader to identify the stopwords from
     * @param fields Selection of fields to calculate stopwords for
     * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
     *                      contain a term, after which the word is considered to be a stop word
     * @throws IOException Can be thrown while reading from the IndexReader
     */
    public QueryAutoStopWordAnalyzer(
        Version matchVersion,
        Analyzer delegate,
        IndexReader indexReader,
        Collection<String> fields,
        float maxPercentDocs) throws IOException {
      this(matchVersion, delegate, indexReader, fields, (int) (indexReader.numDocs() * maxPercentDocs));
    }
  
    /**
     * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for the
     * given selection of fields from terms with a document frequency greater than
     * the given maxDocFreq
     *
     * @param matchVersion Version to be used in {@link StopFilter}
     * @param delegate Analyzer whose TokenStream will be filtered
     * @param indexReader IndexReader to identify the stopwords from
     * @param fields Selection of fields to calculate stopwords for
     * @param maxDocFreq Document frequency terms should be above in order to be stopwords
     * @throws IOException Can be thrown while reading from the IndexReader
     */
    public QueryAutoStopWordAnalyzer(
        Version matchVersion,
        Analyzer delegate,
        IndexReader indexReader,
        Collection<String> fields,
        int maxDocFreq) throws IOException {
      this.matchVersion = matchVersion;
      this.delegate = delegate;
  
      for (String field : fields) {
        Set<String> stopWords = new HashSet<String>();
        String internedFieldName = StringHelper.intern(field);
        TermEnum te = indexReader.terms(new Term(field));
        Term term = te.term();
        while (term != null) {
          if (term.field() != internedFieldName) {
            break;
          }
          if (te.docFreq() > maxDocFreq) {
            stopWords.add(term.text());
          }
          if (!te.next()) {
            break;
          }
          term = te.term();
        }
        stopWordsPerField.put(field, stopWords);
      }
    }
  
    /**
     * Automatically adds stop words for all fields with terms exceeding the defaultMaxDocFreqPercent
     *
     * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
     *               exceed the required document frequency
     * @return The number of stop words identified.
     * @throws IOException
     * @deprecated Stopwords should be calculated at instantiation using
     *             {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader)}
     */
    @Deprecated
    public int addStopWords(IndexReader reader) throws IOException {
      return addStopWords(reader, defaultMaxDocFreqPercent);
    }
  
    /**
     * Automatically adds stop words for all fields with terms exceeding the maxDocFreqPercent
     *
     * @param reader     The {@link IndexReader} which will be consulted to identify potential stop words that
     *                   exceed the required document frequency
     * @param maxDocFreq The maximum number of index documents which can contain a term, after which
     *                   the term is considered to be a stop word
     * @return The number of stop words identified.
     * @throws IOException
     * @deprecated Stopwords should be calculated at instantiation using
     *             {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, int)}
     */
    @Deprecated
    public int addStopWords(IndexReader reader, int maxDocFreq) throws IOException {
      int numStopWords = 0;
      Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
      for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
        String fieldName = iter.next();
        numStopWords += addStopWords(reader, fieldName, maxDocFreq);
      }
      return numStopWords;
    }
  
    /**
     * Automatically adds stop words for all fields with terms exceeding the maxDocFreqPercent
     *
     * @param reader        The {@link IndexReader} which will be consulted to identify potential stop words that
     *                      exceed the required document frequency
     * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
     *                      contain a term, after which the word is considered to be a stop word.
     * @return The number of stop words identified.
     * @throws IOException
     * @deprecated Stowords should be calculated at instantiation using
     *             {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, float)}
     */
    @Deprecated
    public int addStopWords(IndexReader reader, float maxPercentDocs) throws IOException {
      int numStopWords = 0;
      Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
      for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
        String fieldName = iter.next();
        numStopWords += addStopWords(reader, fieldName, maxPercentDocs);
      }
      return numStopWords;
    }
  
    /**
     * Automatically adds stop words for the given field with terms exceeding the maxPercentDocs
     *
     * @param reader         The {@link IndexReader} which will be consulted to identify potential stop words that
     *                       exceed the required document frequency
     * @param fieldName      The field for which stopwords will be added
     * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
     *                       contain a term, after which the word is considered to be a stop word.
     * @return The number of stop words identified.
     * @throws IOException
     * @deprecated Stowords should be calculated at instantiation using
     *             {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, Collection, float)}
     */
    @Deprecated
    public int addStopWords(IndexReader reader, String fieldName, float maxPercentDocs) throws IOException {
      return addStopWords(reader, fieldName, (int) (reader.numDocs() * maxPercentDocs));
    }
  
    /**
     * Automatically adds stop words for the given field with terms exceeding the maxPercentDocs
     *
     * @param reader     The {@link IndexReader} which will be consulted to identify potential stop words that
     *                   exceed the required document frequency
     * @param fieldName  The field for which stopwords will be added
     * @param maxDocFreq The maximum number of index documents which
     *                   can contain a term, after which the term is considered to be a stop word.
     * @return The number of stop words identified.
     * @throws IOException
     * @deprecated Stowords should be calculated at instantiation using
     *             {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, Collection, int)}
     */
    @Deprecated
    public int addStopWords(IndexReader reader, String fieldName, int maxDocFreq) throws IOException {
      HashSet<String> stopWords = new HashSet<String>();
      String internedFieldName = StringHelper.intern(fieldName);
      TermEnum te = reader.terms(new Term(fieldName));
      Term term = te.term();
      while (term != null) {
        if (term.field() != internedFieldName) {
          break;
        }
        if (te.docFreq() > maxDocFreq) {
          stopWords.add(term.text());
        }
        if (!te.next()) {
          break;
        }
        term = te.term();
      }
      stopWordsPerField.put(fieldName, stopWords);
      
      /* if the stopwords for a field are changed,
       * then saved streams for that field are erased.
       */
      @SuppressWarnings("unchecked")
      Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
      if (streamMap != null)
        streamMap.remove(fieldName);
      
      return stopWords.size();
    }
  
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result;
      try {
        result = delegate.reusableTokenStream(fieldName, reader);
      } catch (IOException e) {
        result = delegate.tokenStream(fieldName, reader);
      }
      Set<String> stopWords = stopWordsPerField.get(fieldName);
      if (stopWords != null) {
        result = new StopFilter(matchVersion, result, stopWords);
      }
      return result;
    }
    
    private class SavedStreams {
      /* the underlying stream */
      TokenStream wrapped;
  
      /*
       * when there are no stopwords for the field, refers to wrapped.
       * if there stopwords, it is a StopFilter around wrapped.
       */
      TokenStream withStopFilter;
    }
  
    @SuppressWarnings("unchecked")
    @Override
    public TokenStream reusableTokenStream(String fieldName, Reader reader)
        throws IOException {
      /* map of SavedStreams for each field */
      Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
      if (streamMap == null) {
        streamMap = new HashMap<String, SavedStreams>();
        setPreviousTokenStream(streamMap);
      }
  
      SavedStreams streams = streamMap.get(fieldName);
      if (streams == null) {
        /* an entry for this field does not exist, create one */
        streams = new SavedStreams();
        streamMap.put(fieldName, streams);
        streams.wrapped = delegate.reusableTokenStream(fieldName, reader);
  
        /* if there are any stopwords for the field, save the stopfilter */
        Set<String> stopWords = stopWordsPerField.get(fieldName);
        if (stopWords != null) {
          streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
        } else {
          streams.withStopFilter = streams.wrapped;
        }
  
      } else {
        /*
         * an entry for this field exists, verify the wrapped stream has not
         * changed. if it has not, reuse it, otherwise wrap the new stream.
         */
        TokenStream result = delegate.reusableTokenStream(fieldName, reader);
        if (result == streams.wrapped) {
          /* the wrapped analyzer reused the stream */
        } else {
          /*
           * the wrapped analyzer did not. if there are any stopwords for the
           * field, create a new StopFilter around the new stream
           */
          streams.wrapped = result;
          Set<String> stopWords = stopWordsPerField.get(fieldName);
          if (stopWords != null) {
            streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
          } else {
            streams.withStopFilter = streams.wrapped;
          }
        }
      }
  
      return streams.withStopFilter;
    }
  
    /**
     * Provides information on which stop words have been identified for a field
     *
     * @param fieldName The field for which stop words identified in "addStopWords"
     *                  method calls will be returned
     * @return the stop words identified for a field
     */
    public String[] getStopWords(String fieldName) {
      Set<String> stopWords = stopWordsPerField.get(fieldName);
      return stopWords != null ? stopWords.toArray(new String[stopWords.size()]) : new String[0];
    }
  
    /**
     * Provides information on which stop words have been identified for all fields
     *
     * @return the stop words (as terms)
     */
    public Term[] getStopWords() {
      List<Term> allStopWords = new ArrayList<Term>();
      for (String fieldName : stopWordsPerField.keySet()) {
        Set<String> stopWords = stopWordsPerField.get(fieldName);
        for (String text : stopWords) {
          allStopWords.add(new Term(fieldName, text));
        }
      }
      return allStopWords.toArray(new Term[allStopWords.size()]);
  	}
  
  }
  