content: package org.apache.lucene.index;
  
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import java.io.Closeable;
  import java.io.IOException;
  import java.util.Arrays;
  
  import org.apache.lucene.store.BufferedIndexInput;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.IndexInput;
  import org.apache.lucene.util.ArrayUtil;
  import org.apache.lucene.util.IOUtils;
  
  class TermVectorsReader implements Cloneable, Closeable {
  
    // NOTE: if you make a new format, it must be larger than
    // the current format
    static final int FORMAT_VERSION = 2;
  
    // Changes to speed up bulk merging of term vectors:
    static final int FORMAT_VERSION2 = 3;
  
    // Changed strings to UTF8 with length-in-bytes not length-in-chars
    static final int FORMAT_UTF8_LENGTH_IN_BYTES = 4;
  
    // NOTE: always change this if you switch to a new format!
    static final int FORMAT_CURRENT = FORMAT_UTF8_LENGTH_IN_BYTES;
  
    //The size in bytes that the FORMAT_VERSION will take up at the beginning of each file 
    static final int FORMAT_SIZE = 4;
  
    static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x1;
    static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
    
    private FieldInfos fieldInfos;
  
    private IndexInput tvx;
    private IndexInput tvd;
    private IndexInput tvf;
    private int size;
    private int numTotalDocs;
  
    // The docID offset where our docs begin in the index
    // file.  This will be 0 if we have our own private file.
    private int docStoreOffset;
    
    private final int format;
  
    TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos)
      throws CorruptIndexException, IOException {
      this(d, segment, fieldInfos, BufferedIndexInput.BUFFER_SIZE);
    }
  
    TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize)
      throws CorruptIndexException, IOException {
      this(d, segment, fieldInfos, readBufferSize, -1, 0);
    }
      
    TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize, int docStoreOffset, int size)
      throws CorruptIndexException, IOException {
      boolean success = false;
  
      try {
        String idxName = IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_INDEX_EXTENSION);
        tvx = d.openInput(idxName, readBufferSize);
        format = checkValidFormat(idxName, tvx);
        String fn = IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
        tvd = d.openInput(fn, readBufferSize);
        final int tvdFormat = checkValidFormat(fn, tvd);
        fn = IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_FIELDS_EXTENSION);
        tvf = d.openInput(fn, readBufferSize);
        final int tvfFormat = checkValidFormat(fn, tvf);
  
        assert format == tvdFormat;
        assert format == tvfFormat;
  
        if (format >= FORMAT_VERSION2) {
          numTotalDocs = (int) (tvx.length() >> 4);
        } else {
          assert (tvx.length()-FORMAT_SIZE) % 8 == 0;
          numTotalDocs = (int) (tvx.length() >> 3);
        }
  
        if (-1 == docStoreOffset) {
          this.docStoreOffset = 0;
          this.size = numTotalDocs;
          assert size == 0 || numTotalDocs == size;
        } else {
          this.docStoreOffset = docStoreOffset;
          this.size = size;
          // Verify the file is long enough to hold all of our
          // docs
          assert numTotalDocs >= size + docStoreOffset: "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset;
        }
  
        this.fieldInfos = fieldInfos;
        success = true;
      } finally {
        // With lock-less commits, it's entirely possible (and
        // fine) to hit a FileNotFound exception above. In
        // this case, we want to explicitly close any subset
        // of things that were opened so that we don't have to
        // wait for a GC to do so.
        if (!success) {
          close();
        }
      }
    }
  
    // Used for bulk copy when merging
    IndexInput getTvdStream() {
      return tvd;
    }
  
    // Used for bulk copy when merging
    IndexInput getTvfStream() {
      return tvf;
    }
  
    final private void seekTvx(final int docNum) throws IOException {
      if (format < FORMAT_VERSION2)
        tvx.seek((docNum + docStoreOffset) * 8L + FORMAT_SIZE);
      else
        tvx.seek((docNum + docStoreOffset) * 16L + FORMAT_SIZE);
    }
  
    boolean canReadRawDocs() {
      return format >= FORMAT_UTF8_LENGTH_IN_BYTES;
    }
  
    /** Retrieve the length (in bytes) of the tvd and tvf
     *  entries for the next numDocs starting with
     *  startDocID.  This is used for bulk copying when
     *  merging segments, if the field numbers are
     *  congruent.  Once this returns, the tvf & tvd streams
     *  are seeked to the startDocID. */
    final void rawDocs(int[] tvdLengths, int[] tvfLengths, int startDocID, int numDocs) throws IOException {
  
      if (tvx == null) {
        Arrays.fill(tvdLengths, 0);
        Arrays.fill(tvfLengths, 0);
        return;
      }
  
      // SegmentMerger calls canReadRawDocs() first and should
      // not call us if that returns false.
      if (format < FORMAT_VERSION2)
        throw new IllegalStateException("cannot read raw docs with older term vector formats");
  
      seekTvx(startDocID);
  
      long tvdPosition = tvx.readLong();
      tvd.seek(tvdPosition);
  
      long tvfPosition = tvx.readLong();
      tvf.seek(tvfPosition);
  
      long lastTvdPosition = tvdPosition;
      long lastTvfPosition = tvfPosition;
  
      int count = 0;
      while (count < numDocs) {
        final int docID = docStoreOffset + startDocID + count + 1;
        assert docID <= numTotalDocs;
        if (docID < numTotalDocs)  {
          tvdPosition = tvx.readLong();
          tvfPosition = tvx.readLong();
        } else {
          tvdPosition = tvd.length();
          tvfPosition = tvf.length();
          assert count == numDocs-1;
        }
        tvdLengths[count] = (int) (tvdPosition-lastTvdPosition);
        tvfLengths[count] = (int) (tvfPosition-lastTvfPosition);
        count++;
        lastTvdPosition = tvdPosition;
        lastTvfPosition = tvfPosition;
      }
    }
  
    private int checkValidFormat(String fn, IndexInput in) throws CorruptIndexException, IOException
    {
      int format = in.readInt();
      if (format > FORMAT_CURRENT) {
        throw new IndexFormatTooNewException(in, format, 1, FORMAT_CURRENT);
      }
      return format;
    }
  
    public void close() throws IOException {
      IOUtils.close(tvx, tvd, tvf);
    }
  
    /**
     * 
     * @return The number of documents in the reader
     */
    int size() {
      return size;
    }
  
    public void get(int docNum, String field, TermVectorMapper mapper) throws IOException {
      if (tvx != null) {
        int fieldNumber = fieldInfos.fieldNumber(field);
        //We need to account for the FORMAT_SIZE at when seeking in the tvx
        //We don't need to do this in other seeks because we already have the
        // file pointer
        //that was written in another file
        seekTvx(docNum);
        //System.out.println("TVX Pointer: " + tvx.getFilePointer());
        long tvdPosition = tvx.readLong();
  
        tvd.seek(tvdPosition);
        int fieldCount = tvd.readVInt();
        //System.out.println("Num Fields: " + fieldCount);
        // There are only a few fields per document. We opt for a full scan
        // rather then requiring that they be ordered. We need to read through
        // all of the fields anyway to get to the tvf pointers.
        int number = 0;
        int found = -1;
        for (int i = 0; i < fieldCount; i++) {
          if (format >= FORMAT_VERSION)
            number = tvd.readVInt();
          else
            number += tvd.readVInt();
  
          if (number == fieldNumber)
            found = i;
        }
  
        // This field, although valid in the segment, was not found in this
        // document
        if (found != -1) {
          // Compute position in the tvf file
          long position;
          if (format >= FORMAT_VERSION2)
            position = tvx.readLong();
          else
            position = tvd.readVLong();
          for (int i = 1; i <= found; i++)
            position += tvd.readVLong();
  
          mapper.setDocumentNumber(docNum);
          readTermVector(field, position, mapper);
        } else {
          //System.out.println("Fieldable not found");
        }
      } else {
        //System.out.println("No tvx file");
      }
    }
  
  
  
    /**
     * Retrieve the term vector for the given document and field
     * @param docNum The document number to retrieve the vector for
     * @param field The field within the document to retrieve
     * @return The TermFreqVector for the document and field or null if there is no termVector for this field.
     * @throws IOException if there is an error reading the term vector files
     */ 
    TermFreqVector get(int docNum, String field) throws IOException {
      // Check if no term vectors are available for this segment at all
      ParallelArrayTermVectorMapper mapper = new ParallelArrayTermVectorMapper();
      get(docNum, field, mapper);
  
      return mapper.materializeVector();
    }
  
    // Reads the String[] fields; you have to pre-seek tvd to
    // the right point
    final private String[] readFields(int fieldCount) throws IOException {
      int number = 0;
      String[] fields = new String[fieldCount];
  
      for (int i = 0; i < fieldCount; i++) {
        if (format >= FORMAT_VERSION)
          number = tvd.readVInt();
        else
          number += tvd.readVInt();
  
        fields[i] = fieldInfos.fieldName(number);
      }
  
      return fields;
    }
  
    // Reads the long[] offsets into TVF; you have to pre-seek
    // tvx/tvd to the right point
    final private long[] readTvfPointers(int fieldCount) throws IOException {
      // Compute position in the tvf file
      long position;
      if (format >= FORMAT_VERSION2)
        position = tvx.readLong();
      else
        position = tvd.readVLong();
  
      long[] tvfPointers = new long[fieldCount];
      tvfPointers[0] = position;
  
      for (int i = 1; i < fieldCount; i++) {
        position += tvd.readVLong();
        tvfPointers[i] = position;
      }
  
      return tvfPointers;
    }
  
    /**
     * Return all term vectors stored for this document or null if the could not be read in.
     * 
     * @param docNum The document number to retrieve the vector for
     * @return All term frequency vectors
     * @throws IOException if there is an error reading the term vector files 
     */
    TermFreqVector[] get(int docNum) throws IOException {
      TermFreqVector[] result = null;
      if (tvx != null) {
        //We need to offset by
        seekTvx(docNum);
        long tvdPosition = tvx.readLong();
  
        tvd.seek(tvdPosition);
        int fieldCount = tvd.readVInt();
  
        // No fields are vectorized for this document
        if (fieldCount != 0) {
          final String[] fields = readFields(fieldCount);
          final long[] tvfPointers = readTvfPointers(fieldCount);
          result = readTermVectors(docNum, fields, tvfPointers);
        }
      } else {
        //System.out.println("No tvx file");
      }
      return result;
    }
  
    public void get(int docNumber, TermVectorMapper mapper) throws IOException {
      // Check if no term vectors are available for this segment at all
      if (tvx != null) {
        //We need to offset by
  
        seekTvx(docNumber);
        long tvdPosition = tvx.readLong();
  
        tvd.seek(tvdPosition);
        int fieldCount = tvd.readVInt();
  
        // No fields are vectorized for this document
        if (fieldCount != 0) {
          final String[] fields = readFields(fieldCount);
          final long[] tvfPointers = readTvfPointers(fieldCount);
          mapper.setDocumentNumber(docNumber);
          readTermVectors(fields, tvfPointers, mapper);
        }
      } else {
        //System.out.println("No tvx file");
      }
    }
  
  
    private SegmentTermVector[] readTermVectors(int docNum, String fields[], long tvfPointers[])
            throws IOException {
      SegmentTermVector res[] = new SegmentTermVector[fields.length];
      for (int i = 0; i < fields.length; i++) {
        ParallelArrayTermVectorMapper mapper = new ParallelArrayTermVectorMapper();
        mapper.setDocumentNumber(docNum);
        readTermVector(fields[i], tvfPointers[i], mapper);
        res[i] = (SegmentTermVector) mapper.materializeVector();
      }
      return res;
    }
  
    private void readTermVectors(String fields[], long tvfPointers[], TermVectorMapper mapper)
            throws IOException {
      for (int i = 0; i < fields.length; i++) {
        readTermVector(fields[i], tvfPointers[i], mapper);
      }
    }
  
  
    /**
     * 
     * @param field The field to read in
     * @param tvfPointer The pointer within the tvf file where we should start reading
     * @param mapper The mapper used to map the TermVector
     * @throws IOException
     */ 
    private void readTermVector(String field, long tvfPointer, TermVectorMapper mapper)
            throws IOException {
  
      // Now read the data from specified position
      //We don't need to offset by the FORMAT here since the pointer already includes the offset
      tvf.seek(tvfPointer);
  
      int numTerms = tvf.readVInt();
      //System.out.println("Num Terms: " + numTerms);
      // If no terms - return a constant empty termvector. However, this should never occur!
      if (numTerms == 0) 
        return;
      
      boolean storePositions;
      boolean storeOffsets;
      
      if (format >= FORMAT_VERSION){
        byte bits = tvf.readByte();
        storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
        storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
      }
      else{
        tvf.readVInt();
        storePositions = false;
        storeOffsets = false;
      }
      mapper.setExpectations(field, numTerms, storeOffsets, storePositions);
      int start = 0;
      int deltaLength = 0;
      int totalLength = 0;
      byte[] byteBuffer;
      char[] charBuffer;
      final boolean preUTF8 = format < FORMAT_UTF8_LENGTH_IN_BYTES;
  
      // init the buffers
      if (preUTF8) {
        charBuffer = new char[10];
        byteBuffer = null;
      } else {
        charBuffer = null;
        byteBuffer = new byte[20];
      }
  
      for (int i = 0; i < numTerms; i++) {
        start = tvf.readVInt();
        deltaLength = tvf.readVInt();
        totalLength = start + deltaLength;
  
        final String term;
        
        if (preUTF8) {
          // Term stored as java chars
          if (charBuffer.length < totalLength) {
            charBuffer = ArrayUtil.grow(charBuffer, totalLength);
          }
          tvf.readChars(charBuffer, start, deltaLength);
          term = new String(charBuffer, 0, totalLength);
        } else {
          // Term stored as utf8 bytes
          if (byteBuffer.length < totalLength) {
            byteBuffer = ArrayUtil.grow(byteBuffer, totalLength);
          }
          tvf.readBytes(byteBuffer, start, deltaLength);
          term = new String(byteBuffer, 0, totalLength, "UTF-8");
        }
        int freq = tvf.readVInt();
        int [] positions = null;
        if (storePositions) { //read in the positions
          //does the mapper even care about positions?
          if (mapper.isIgnoringPositions() == false) {
            positions = new int[freq];
            int prevPosition = 0;
            for (int j = 0; j < freq; j++)
            {
              positions[j] = prevPosition + tvf.readVInt();
              prevPosition = positions[j];
            }
          } else {
            //we need to skip over the positions.  Since these are VInts, I don't believe there is anyway to know for sure how far to skip
            //
            for (int j = 0; j < freq; j++)
            {
              tvf.readVInt();
            }
          }
        }
        TermVectorOffsetInfo[] offsets = null;
        if (storeOffsets) {
          //does the mapper even care about offsets?
          if (mapper.isIgnoringOffsets() == false) {
            offsets = new TermVectorOffsetInfo[freq];
            int prevOffset = 0;
            for (int j = 0; j < freq; j++) {
              int startOffset = prevOffset + tvf.readVInt();
              int endOffset = startOffset + tvf.readVInt();
              offsets[j] = new TermVectorOffsetInfo(startOffset, endOffset);
              prevOffset = endOffset;
            }
          } else {
            for (int j = 0; j < freq; j++){
              tvf.readVInt();
              tvf.readVInt();
            }
          }
        }
        mapper.map(term, freq, offsets, positions);
      }
    }
  
    @Override
    protected Object clone() throws CloneNotSupportedException {
      
      final TermVectorsReader clone = (TermVectorsReader) super.clone();
  
      // These are null when a TermVectorsReader was created
      // on a segment that did not have term vectors saved
      if (tvx != null && tvd != null && tvf != null) {
        clone.tvx = (IndexInput) tvx.clone();
        clone.tvd = (IndexInput) tvd.clone();
        clone.tvf = (IndexInput) tvf.clone();
      }
      
      return clone;
    }
  }
  
  
  /**
   * Models the existing parallel array structure
   */
  class ParallelArrayTermVectorMapper extends TermVectorMapper
  {
  
    private String[] terms;
    private int[] termFreqs;
    private int positions[][];
    private TermVectorOffsetInfo offsets[][];
    private int currentPosition;
    private boolean storingOffsets;
    private boolean storingPositions;
    private String field;
  
    @Override
    public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
      this.field = field;
      terms = new String[numTerms];
      termFreqs = new int[numTerms];
      this.storingOffsets = storeOffsets;
      this.storingPositions = storePositions;
      if(storePositions)
        this.positions = new int[numTerms][];
      if(storeOffsets)
        this.offsets = new TermVectorOffsetInfo[numTerms][];
    }
  
    @Override
    public void map(String term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
      terms[currentPosition] = term;
      termFreqs[currentPosition] = frequency;
      if (storingOffsets)
      {
        this.offsets[currentPosition] = offsets;
      }
      if (storingPositions)
      {
        this.positions[currentPosition] = positions; 
      }
      currentPosition++;
    }
  
    /**
     * Construct the vector
     * @return The {@link TermFreqVector} based on the mappings.
     */
    public TermFreqVector materializeVector() {
      SegmentTermVector tv = null;
      if (field != null && terms != null) {
        if (storingPositions || storingOffsets) {
          tv = new SegmentTermPositionVector(field, terms, termFreqs, positions, offsets);
        } else {
          tv = new SegmentTermVector(field, terms, termFreqs);
        }
      }
      return tv;
    }
  }
  