content: package org.apache.solr.request.uninverted;
  
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.PriorityQueue;
  import java.util.zip.CRC32;
  
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.index.TermDocs;
  import org.apache.solr.request.mdrill.MdrillUtils;
  import org.apache.solr.request.uninverted.UnInvertedFieldUtils.FieldDatatype;
  import org.apache.solr.request.uninverted.UnInvertedFieldUtils.MixTermInfo;
  import org.apache.solr.schema.IndexSchema;
  import org.apache.solr.search.BitDocSet;
  import org.apache.solr.search.DocIterator;
  import org.apache.solr.search.DocSet;
  import org.apache.solr.search.DocSetCollector;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  
  /**
   * 历史的旧方法，通过遍历倒排表来实现，效率不高
   * @author yannian.mu
   *
   */
  public class MakeUnivertedFieldByIndex {
  	public static Logger log = LoggerFactory.getLogger(MakeUnivertedFieldByIndex.class);
  
  	private static int SKIP_STEP = 32;
  	private static int SKIP_MIN = 64;
  	private static int MAX_SKIP_COUNT = 102400;
  	
  	private static long TD_MAX_CMP_COUNT = 64000000l;
  	private static int TD_LIMIT_MAX = 10240;
  	private static int TD_LIMIT_MIN = 8;
  	
  	private int[] docs = new int[1000];
  	private int[] freqs = new int[1000];
  	
  	private UnInvertedField uni;
  
  	public MakeUnivertedFieldByIndex(UnInvertedField uni) throws IOException {
  		this.uni=uni;
  	}
  	
  	public void makeInit(BitDocSet baseAdvanceDocs,String field, IndexSchema schema,IndexReader reader) throws IOException
  	{
  		uni.init(field, reader, schema);
  		uni.baseAdvanceDocs=UnInvertedField.ajustBase(48,baseAdvanceDocs, reader);
  
  		if(this.uni.checkEmpty())
  		{
  			return ;
  		}
  		
  		log.info(" makeInit  begin " + this.uni.field + " field " +",baseAdvanceDocs="+(this.uni.baseAdvanceDocs==null?"null":this.uni.baseAdvanceDocs.size())+"@"+(baseAdvanceDocs==null?"null":baseAdvanceDocs.size()));
  			
  		TermNumEnumerator te = uni.ti.getEnumerator(reader);
  
  		int maxDoc = reader.maxDoc();
  
  		this.uni.startRamDocValue(maxDoc, reader, true);
  		
  		int limitsize=this.getLimitSize();
  
  		PriorityQueue<MixTermInfo> termDocslist=new PriorityQueue<MixTermInfo>(limitsize,Collections.reverseOrder(UnInvertedFieldUtils.TD_CMP));
  
  		TermDocs tdreader=reader.termDocs(1024);
  		int maxTermNum=0;
  		for (;;) {
  			Term t = te.term();
  			if (t == null) {
  				break;
  			}
  
  			int termNum = te.getTermNumber();
  			
  			if(termNum%100000==0)
  			{
  				log.info("termsInverted " +termNum+"@"+ this.uni.field + ",limitsize=" + limitsize);
  			}
  			
  			int df=te.docFreq();
  			
  			if(!this.isFinish())
  			{
  				TermDocs td = te.getTermDocs();
  				if(df<=SKIP_MIN||this.uni.baseAdvanceDocs==null)
  				{
  					td.seek(te);
  					this.set_Doc2TermNum_NonSkip(td, docs, freqs, termNum,true,maxDoc);
  				} else {
  
  					MixTermInfo cl = new MixTermInfo(df, termNum, tdreader,	new Term(t.field(), t.text()));
  					if (termDocslist.size() < limitsize) {
  						termDocslist.add(cl);
  					} else {
  						MixTermInfo peek = termDocslist.peek();
  						if (UnInvertedFieldUtils.TD_CMP.compare(peek, cl) > 0 && cl.getCount() / (peek.getCount() + 1) > 1.5) {
  							termDocslist.add(cl);
  							MixTermInfo cl_old = termDocslist.poll();
  							this.set_Doc2TermNum_NonSkip(cl_old.getTd(), docs,freqs, cl_old.getTermNum(), true, maxDoc);
  						} else {
  							td.seek(te);
  							this.set_Doc2TermNum_NonSkip(td, docs, freqs,termNum, true, maxDoc);
  						}
  					}
  				}
  			}
  			
  			maxTermNum=Math.max(maxTermNum, termNum);
  
  			this.setTermNumValue(t, termNum);
  			te.next();
  		}
  		
  		this.PriorityQueue_skip_set(termDocslist, maxDoc,true);
  
  
  		this.uni.endRamDocValue(true,maxTermNum);
  
  		tdreader.close();
  		te.close();
  
  		this.uni.tnr = this.uni.ramDocValue.getDocReader();
  	
  		this.setTdIndex_NULL();
  	}
  	
  	
  	public void setTdIndex_NULL() throws IOException {
  		if (this.uni.baseAdvanceDocs == null) {
  			return;
  		}
  
  		log.info("setTdIndex_NULL :"+this.uni.baseAdvanceDocs.size());
  		DocIterator iter = this.uni.baseAdvanceDocs.iterator();
  		while (iter.hasNext()) {
  			int doc = iter.nextDoc();
  			this.uni.bits.add(doc);
  			this.uni.markDocTm(doc, this.uni.getNullTm(),false);
  		}
  		
  		this.uni.baseAdvanceDocs=null;
  	}
  	
  	
  	public void addDoclist(BitDocSet baseAdvanceDocs,String field, 
  			IndexReader reader) throws IOException {
  		
  		if (uni.checkEmpty()) {
  			return ;
  		}
  
  		BitDocSet tmp=null;
  		if(baseAdvanceDocs!=null)
  		{
  			tmp=(BitDocSet) baseAdvanceDocs.andNot(this.uni.bits);
  
  			if(tmp!=null&&tmp.size()<=0)
  			{
  				return ;
  			}
  		}
  	
  		this.uni.baseAdvanceDocs=UnInvertedField.ajustBase(48,tmp, reader);
  		TermNumEnumerator te = uni.ti.getEnumerator(reader);
  
  		log.info("addDoclist start " + this.uni.field +",baseAdvanceDocs="+(this.uni.baseAdvanceDocs==null?"null":this.uni.baseAdvanceDocs.size())+"@"+(baseAdvanceDocs==null?"null":baseAdvanceDocs.size()));
  
  		int maxDoc=reader.maxDoc();
  		int limitsize=this.getLimitSize();
  
  		PriorityQueue<MixTermInfo> termDocslist=new PriorityQueue<MixTermInfo>(limitsize,Collections.reverseOrder(UnInvertedFieldUtils.TD_CMP));
  		TermDocs tdreader=reader.termDocs(1024);
  
  		for (;;) {
  			Term t = te.term();
  			if (t == null) {
  				break;
  			}
  
  			if (this.isFinish()) {
  				break;
  			}
  
  			int termNum = te.getTermNumber();
  
  			if (termNum%10000==0) {
  				log.info("termsInverted " +termNum+"@"+ this.uni.field + ",limitsize=" + limitsize);
  			}
  			TermDocs td = te.getTermDocs();
  
  			int df = te.docFreq();
  			if (df <= SKIP_MIN || this.uni.baseAdvanceDocs == null) {
  				td.seek(te);
  				this.set_Doc2TermNum_NonSkip(td, docs, freqs, termNum, false, maxDoc);
  			} else {
  				MixTermInfo cl = new MixTermInfo(df, termNum, tdreader,	new Term(t.field(), t.text()));
  				if (termDocslist.size() < limitsize) {
  					termDocslist.add(cl);
  				} else {
  					MixTermInfo peek = termDocslist.peek();
  					if (UnInvertedFieldUtils.TD_CMP.compare(peek, cl) > 0 && cl.getCount() / (peek.getCount() + 1) > 1.5) {
  						termDocslist.add(cl);
  						MixTermInfo cl_old = termDocslist.poll();
  						this.set_Doc2TermNum_NonSkip(cl_old.getTd(), docs,	freqs, cl_old.getTermNum(), false, maxDoc);
  					} else {
  						td.seek(te);
  						this.set_Doc2TermNum_NonSkip(td, docs, freqs, termNum, false, maxDoc);
  					}
  				}
  			}
  			te.next();
  		}
  		
  		this.PriorityQueue_skip_set(termDocslist, maxDoc,false);
  		
  		te.close();
  		this.setTdIndex_NULL();
  	}
  	
  	private boolean isFinish()
  	{
  		if(this.uni.baseAdvanceDocs==null)
  		{
  			return false;
  		}
  		
  		return this.uni.baseAdvanceDocs.size()<=0;
  	}
  	
  	private void cleanBase(DocSetCollector collect)
  	{
  		DocSet docsit=collect.getDocSet();
  		DocIterator toremove = docsit.iterator();
  		while (toremove.hasNext()) {
  			int doc = toremove.nextDoc();
  			this.uni.baseAdvanceDocs.clear(doc);
  		}
  		
  		
  
  	}
  
  	private int getLimitSize()
  	{
  		int limitsize=TD_LIMIT_MAX;
  		if(this.uni.baseAdvanceDocs!=null)
  		{
  			limitsize=(int) Math.min(limitsize, TD_MAX_CMP_COUNT/(1+this.uni.baseAdvanceDocs.size()));
  		}
  		limitsize=Math.max(limitsize, TD_LIMIT_MIN);
  		return limitsize;
  	}
  	
  	private void set_Doc2TermNum_NonSkip(TermDocs td,int[] docs,int[] freqs,int termNum,boolean isinit,int maxDoc) throws IOException
  	{
  		if(this.uni.baseAdvanceDocs==null)
  		{
  			for (;;) {
  				int n = td.read(docs, freqs);
  				if (n <= 0) {
  					break;
  				}
  				for (int i = 0; i < n; i++) {
  					int docid=docs[i];
  					this.uni.bits.add(docid);
  					this.uni.markDocTm(docid, termNum, isinit);
  				}
  			}
  			return ;
  		}
  		
  		DocSetCollector collect=new DocSetCollector(10240, maxDoc);
  		for (;;) {
  			int n = td.read(docs, freqs);
  			if (n <= 0) {
  				break;
  			}
  			for (int i = 0; i < n; i++) {
  				int docid=docs[i];
  				collect.collect(docid);
  				this.uni.bits.add(docid);
  				this.uni.markDocTm(docid, termNum, isinit);
  			}
  		}
  
  		this.cleanBase(collect);
  
  	}
  
  	private int set_Doc2TermNum_Skip(TermDocs td,int[] docs,int[] freqs,int termNum,boolean isinit,int maxDoc) throws IOException
  	{
  		int skipcount=0;
  			
  		DocSetCollector collect=new DocSetCollector(10240, maxDoc);
  		DocIterator iter = this.uni.baseAdvanceDocs.iterator();
  		int doc=-1;
  		int baseDoc=-1;
  		
  		while (iter.hasNext()) {
  			doc = iter.nextDoc();
  			if(doc<baseDoc)
  			{
  				continue;
  			}
  			
  			
  			if(doc>baseDoc)
  			{
  				int diff=doc-baseDoc;
  				if(diff>=SKIP_STEP)
  				{
  					if(baseDoc>=0)
  					{
  						skipcount++;
  					}
  					baseDoc=UnInvertedFieldUtils.advance(td, doc);
  					if(UnInvertedFieldUtils.NO_MORE_DOCS==baseDoc)
  					{
  						break;
  					}
  				}else{
  					boolean is_no_more_docs=false;
  					for(int i=0;i<=SKIP_STEP;i++)
  					{
  						if(td.next())
  						{
  							baseDoc=td.doc();
  							if(baseDoc>=doc)
  							{
  								break;
  							}
  							collect.collect(baseDoc);
  							this.uni.bits.add(baseDoc);
  							this.uni.markDocTm(baseDoc, termNum, isinit);
  							
  						}else{
  							is_no_more_docs=true;
  							break;
  						}
  					}
  					if(is_no_more_docs)
  					{
  						break;
  					}
  				}
  			}
  			
  			if(baseDoc==doc)
  			{
  				this.uni.bits.add(doc);
  				collect.collect(doc);
  				this.uni.markDocTm(doc, termNum, isinit);
  			}
  		}
  		
  		
  		this.cleanBase(collect);
  		
  		return skipcount;
  	}
  	
  	private int PriorityQueue_skip_set(PriorityQueue<MixTermInfo> termDocslist,int maxDoc,boolean isinit) throws IOException
  	{
  		int skipcount=0;
  		ArrayList<MixTermInfo> sorted=new ArrayList<MixTermInfo>(termDocslist.size());
  		sorted.addAll(termDocslist);
  		Collections.sort(sorted, UnInvertedFieldUtils.TD_CMP_TM);
  		for(MixTermInfo cl:sorted)
  		{
  			if(this.isFinish())
  			{
  				break;
  			}
  
  			if(skipcount>MAX_SKIP_COUNT)
  			{
  				this.set_Doc2TermNum_NonSkip(cl.getTd(), docs, freqs,cl.getTermNum(),isinit,maxDoc);
  			}else{
  				skipcount+=this.set_Doc2TermNum_Skip(cl.getTd(), docs, freqs, cl.getTermNum(),isinit,maxDoc);
  			}
  		
  		}
  		
  		return skipcount;
  	}
  	
  	private void setTermNumValue(Term t,int termNum)
  	{
  		if (this.uni.fieldDataType == FieldDatatype.d_long) {
  			this.uni.setTmValueLong(termNum,Long.parseLong(this.uni.ft.indexedToReadable(t.text()))) ;
  		} else if (this.uni.fieldDataType == FieldDatatype.d_double) {
  			this.uni.setTmValueDouble(termNum, MdrillUtils.ParseDouble(this.uni.ft.indexedToReadable(t.text())));
  		} else if (this.uni.fieldDataType == FieldDatatype.d_string) {// for dist
  			CRC32 crc32 = new CRC32();
  			crc32.update(new String(this.uni.ft.indexedToReadable(t.text())).getBytes());
  			this.uni.setTmValueLong(termNum, crc32.getValue());
  		}
  	}
  
  }
  