content: package org.apache.lucene.search;
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  import java.io.IOException;
  
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.index.TermDocs;
  import org.apache.lucene.index.TermEnum;
  import org.apache.lucene.util.FixedBitSet;
  
  public class DuplicateFilter extends Filter
  {
  	
  	String fieldName;
  	
  	/**
  	 * KeepMode determines which document id to consider as the master, all others being 
  	 * identified as duplicates. Selecting the "first occurrence" can potentially save on IO.
  	 */
  	int keepMode=KM_USE_FIRST_OCCURRENCE;
  	public static final int KM_USE_FIRST_OCCURRENCE=1;
  	public static final int KM_USE_LAST_OCCURRENCE=2;
  	
  	/**
  	 * "Full" processing mode starts by setting all bits to false and only setting bits
  	 * for documents that contain the given field and are identified as none-duplicates. 
  
  	 * "Fast" processing sets all bits to true then unsets all duplicate docs found for the
  	 * given field. This approach avoids the need to read TermDocs for terms that are seen 
  	 * to have a document frequency of exactly "1" (i.e. no duplicates). While a potentially 
  	 * faster approach , the downside is that bitsets produced will include bits set for 
  	 * documents that do not actually contain the field given.
  	 * 
  	 */
  	int processingMode=PM_FULL_VALIDATION;
  	public static final int PM_FULL_VALIDATION=1;
  	public static final int PM_FAST_INVALIDATION=2;
  	
  
  	
  	public DuplicateFilter(String fieldName)
  	{
  		this(fieldName, KM_USE_LAST_OCCURRENCE,PM_FULL_VALIDATION);
  	}
  	
  
  	public DuplicateFilter(String fieldName, int keepMode, int processingMode)
  	{
  		this.fieldName = fieldName;
  		this.keepMode = keepMode;
  		this.processingMode = processingMode;
  	}
  
    @Override
    public DocIdSet getDocIdSet(IndexReader reader) throws IOException
  	{
  		if(processingMode==PM_FAST_INVALIDATION)
  		{
  			return fastBits(reader);
  		}
  		else
  		{
  			return correctBits(reader);
  		}
  	}
  	
    private FixedBitSet correctBits(IndexReader reader) throws IOException
  	{
  		
      FixedBitSet bits=new FixedBitSet(reader.maxDoc()); //assume all are INvalid
  		Term startTerm=new Term(fieldName);
  		TermEnum te = reader.terms(startTerm);
  		if(te!=null)
  		{
  			Term currTerm=te.term();
  			while((currTerm!=null)&&(currTerm.field()==startTerm.field())) //term fieldnames are interned
  			{
  				int lastDoc=-1;
  				//set non duplicates
  				TermDocs td = reader.termDocs(currTerm);
  				if(td.next())
  				{
  					if(keepMode==KM_USE_FIRST_OCCURRENCE)
  					{
  						bits.set(td.doc());
  					}
  					else
  					{
  						do
  						{
  							lastDoc=td.doc();
  						}while(td.next());
  						bits.set(lastDoc);
  					}
  				}
  				if(!te.next())
  				{
  					break;
  				}
  				currTerm=te.term();
  			}
  		}
  		return bits;
  	}
  	
    private FixedBitSet fastBits(IndexReader reader) throws IOException
  	{
  		
      FixedBitSet bits=new FixedBitSet(reader.maxDoc());
  		bits.set(0,reader.maxDoc()); //assume all are valid
  		Term startTerm=new Term(fieldName);
  		TermEnum te = reader.terms(startTerm);
  		if(te!=null)
  		{
  			Term currTerm=te.term();
  			
  			while((currTerm!=null)&&(currTerm.field()==startTerm.field())) //term fieldnames are interned
  			{
  				if(te.docFreq()>1)
  				{
  					int lastDoc=-1;
  					//unset potential duplicates
  					TermDocs td = reader.termDocs(currTerm);
  					td.next();
  					if(keepMode==KM_USE_FIRST_OCCURRENCE)
  					{
  						td.next();
  					}
  					do
  					{
  						lastDoc=td.doc();
              bits.clear(lastDoc);
  					}while(td.next());
  					if(keepMode==KM_USE_LAST_OCCURRENCE)
  					{
  						//restore the last bit
  						bits.set(lastDoc);
  					}					
  				}
  				if(!te.next())
  				{
  					break;
  				}
  				currTerm=te.term();
  			}
  		}
  		return bits;
  	}
  
  	public String getFieldName()
  	{
  		return fieldName;
  	}
  
  
  	public void setFieldName(String fieldName)
  	{
  		this.fieldName = fieldName;
  	}
  
  
  	public int getKeepMode()
  	{
  		return keepMode;
  	}
  
  
  	public void setKeepMode(int keepMode)
  	{
  		this.keepMode = keepMode;
  	}
  
  
  	@Override
  	public boolean equals(Object obj)
  	{
  		if(this == obj)
  			return true;
  		if((obj == null) || (obj.getClass() != this.getClass()))
  			return false;
  		DuplicateFilter other = (DuplicateFilter)obj;
  		return keepMode == other.keepMode &&
  		processingMode == other.processingMode &&
  			(fieldName == other.fieldName || (fieldName != null && fieldName.equals(other.fieldName)));
  	}
  
  
  
  	@Override
  	public int hashCode()
  	{
  		int hash = 217;
  		hash = 31 * hash + keepMode;
  		hash = 31 * hash + processingMode;
  		hash = 31 * hash + fieldName.hashCode();
  		return hash;	
  	}
  
  
  	public int getProcessingMode()
  	{
  		return processingMode;
  	}
  
  
  	public void setProcessingMode(int processingMode)
  	{
  		this.processingMode = processingMode;
  	}
  	
  	
  
  }
  