content: package org.apache.lucene.index;
  
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import java.io.IOException;
  import java.io.Reader;
  import org.apache.lucene.document.Fieldable;
  import org.apache.lucene.analysis.TokenStream;
  import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
  import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
  
  /**
   * Holds state for inverting all occurrences of a single
   * field in the document.  This class doesn't do anything
   * itself; instead, it forwards the tokens produced by
   * analysis to its own consumer
   * (InvertedDocConsumerPerField).  It also interacts with an
   * endConsumer (InvertedDocEndConsumerPerField).
   */
  
  final class DocInverterPerField extends DocFieldConsumerPerField {
  
    final private DocInverterPerThread perThread;
    final private FieldInfo fieldInfo;
    final InvertedDocConsumerPerField consumer;
    final InvertedDocEndConsumerPerField endConsumer;
    final DocumentsWriter.DocState docState;
    final FieldInvertState fieldState;
  
    public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo) {
      this.perThread = perThread;
      this.fieldInfo = fieldInfo;
      docState = perThread.docState;
      fieldState = perThread.fieldState;
      this.consumer = perThread.consumer.addField(this, fieldInfo);
      this.endConsumer = perThread.endConsumer.addField(this, fieldInfo);
    }
  
    @Override
    void abort() {
      try {
        consumer.abort();
      } finally {
        endConsumer.abort();
      }
    }
  
    @Override
    public void processFields(final Fieldable[] fields,
                              final int count) throws IOException {
  
      fieldState.reset(docState.doc.getBoost());
  
      final int maxFieldLength = docState.maxFieldLength;
  
      final boolean doInvert = consumer.start(fields, count);
  
      for(int i=0;i<count;i++) {
  
        final Fieldable field = fields[i];
  
        // TODO FI: this should be "genericized" to querying
        // consumer if it wants to see this particular field
        // tokenized.
        if (field.isIndexed() && doInvert) {
          
          if (i > 0)
            fieldState.position += docState.analyzer == null ? 0 : docState.analyzer.getPositionIncrementGap(fieldInfo.name);
  
          if (!field.isTokenized()) {		  // un-tokenized field
            String stringValue = field.stringValue();
            final int valueLength = stringValue.length();
            perThread.singleToken.reinit(stringValue, 0, valueLength);
            fieldState.attributeSource = perThread.singleToken;
            consumer.start(field);
  
            boolean success = false;
            try {
              consumer.add();
              success = true;
            } finally {
              if (!success)
                docState.docWriter.setAborting();
            }
            fieldState.offset += valueLength;
            fieldState.length++;
            fieldState.position++;
          } else {                                  // tokenized field
            final TokenStream stream;
            final TokenStream streamValue = field.tokenStreamValue();
  
            if (streamValue != null) 
              stream = streamValue;
            else {
              // the field does not have a TokenStream,
              // so we have to obtain one from the analyzer
              final Reader reader;			  // find or make Reader
              final Reader readerValue = field.readerValue();
  
              if (readerValue != null)
                reader = readerValue;
              else {
                String stringValue = field.stringValue();
                if (stringValue == null)
                  throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
                perThread.stringReader.init(stringValue);
                reader = perThread.stringReader;
              }
            
              // Tokenize field and add to postingTable
              stream = docState.analyzer.reusableTokenStream(fieldInfo.name, reader);
            }
  
            // reset the TokenStream to the first token
            stream.reset();
  
            final int startLength = fieldState.length;
            
            try {
              boolean hasMoreTokens = stream.incrementToken();
  
              fieldState.attributeSource = stream;
  
              OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
              PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class);
              
              consumer.start(field);
              
              for(;;) {
  
                // If we hit an exception in stream.next below
                // (which is fairly common, eg if analyzer
                // chokes on a given document), then it's
                // non-aborting and (above) this one document
                // will be marked as deleted, but still
                // consume a docID
                
                if (!hasMoreTokens) break;
                
                final int posIncr = posIncrAttribute.getPositionIncrement();
                fieldState.position += posIncr;
                if (fieldState.position > 0) {
                  fieldState.position--;
                }
  
                if (posIncr == 0)
                  fieldState.numOverlap++;
  
                boolean success = false;
                try {
                  // If we hit an exception in here, we abort
                  // all buffered documents since the last
                  // flush, on the likelihood that the
                  // internal state of the consumer is now
                  // corrupt and should not be flushed to a
                  // new segment:
                  consumer.add();
                  success = true;
                } finally {
                  if (!success)
                    docState.docWriter.setAborting();
                }
                fieldState.position++;
                if (++fieldState.length >= maxFieldLength) {
                  if (docState.infoStream != null)
                    docState.infoStream.println("maxFieldLength " +maxFieldLength+ " reached for field " + fieldInfo.name + ", ignoring following tokens");
                  break;
                }
  
                hasMoreTokens = stream.incrementToken();
              }
              // trigger streams to perform end-of-stream operations
              stream.end();
              
              fieldState.offset += offsetAttribute.endOffset();
            } finally {
              stream.close();
            }
          }
  
          fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
          fieldState.boost *= field.getBoost();
        }
  
        // LUCENE-2387: don't hang onto the field, so GC can
        // reclaim
        fields[i] = null;
      }
  
      consumer.finish();
      endConsumer.finish();
    }
  }
  