content: package org.apache.lucene.index;
  
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.IndexInput;
  import org.apache.lucene.store.IndexOutput;
  import org.apache.lucene.store.ChecksumIndexOutput;
  import org.apache.lucene.store.ChecksumIndexInput;
  import org.apache.lucene.store.NoSuchDirectoryException;
  import org.apache.lucene.util.IOUtils;
  import org.apache.lucene.util.ThreadInterruptedException;
  
  import com.alimama.mdrill.hdfsDirectory.Descriptor;
  
  import java.io.FileNotFoundException;
  import java.io.IOException;
  import java.io.PrintStream;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.HashSet;
  import java.util.HashMap;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  
  /**
   * A collection of segmentInfo objects with methods for operating on
   * those segments in relation to the file system.
   * 
   * @lucene.experimental
   */
  public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
      private static final Log logger = LogFactory.getLog(SegmentInfos.class);
  
    /** The file format version, a negative number. */
    /* Works since counter, the old 1st entry, is always >= 0 */
    public static final int FORMAT = -1;
  
    /** This format adds details used for lockless commits.  It differs
     * slightly from the previous format in that file names
     * are never re-used (write once).  Instead, each file is
     * written to the next generation.  For example,
     * segments_1, segments_2, etc.  This allows us to not use
     * a commit lock.  See <a
     * href="http://lucene.apache.org/java/docs/fileformats.html">file
     * formats</a> for details.
     */
    public static final int FORMAT_LOCKLESS = -2;
  
    /** This format adds a "hasSingleNormFile" flag into each segment info.
     * See <a href="http://issues.apache.org/jira/browse/LUCENE-756">LUCENE-756</a>
     * for details.
     */
    public static final int FORMAT_SINGLE_NORM_FILE = -3;
  
    /** This format allows multiple segments to share a single
     * vectors and stored fields file. */
    public static final int FORMAT_SHARED_DOC_STORE = -4;
  
    /** This format adds a checksum at the end of the file to
     *  ensure all bytes were successfully written. */
    public static final int FORMAT_CHECKSUM = -5;
  
    /** This format adds the deletion count for each segment.
     *  This way IndexWriter can efficiently report numDocs(). */
    public static final int FORMAT_DEL_COUNT = -6;
  
    /** This format adds the boolean hasProx to record if any
     *  fields in the segment store prox information (ie, have
     *  omitTermFreqAndPositions==false) */
    public static final int FORMAT_HAS_PROX = -7;
  
    /** This format adds optional commit userData (String) storage. */
    public static final int FORMAT_USER_DATA = -8;
  
    /** This format adds optional per-segment String
     *  diagnostics storage, and switches userData to Map */
    public static final int FORMAT_DIAGNOSTICS = -9;
  
    /** Each segment records whether it has term vectors */
    public static final int FORMAT_HAS_VECTORS = -10;
  
    /** Each segment records the Lucene version that created it. */
    public static final int FORMAT_3_1 = -11;
    
    /* This must always point to the most recent file format. */
    public static final int CURRENT_FORMAT = FORMAT_3_1;
  
    public static final int FORMAT_MINIMUM = FORMAT;
    public static final int FORMAT_MAXIMUM = CURRENT_FORMAT;
    
    public int counter = 0;    // used to name new segments
    /**
     * counts how often the index has been changed by adding or deleting docs.
     * starting with the current time in milliseconds forces to create unique version numbers.
     */
    long version = System.currentTimeMillis();
  
    private long generation = 0;     // generation of the "segments_N" for the next commit
    private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
                                     // or wrote; this is normally the same as generation except if
                                     // there was an IOException that had interrupted a commit
  
    private Map<String,String> userData = Collections.<String,String>emptyMap();       // Opaque Map<String, String> that user can specify during IndexWriter.commit
  
    private int format;
    
    private List<SegmentInfo> segments = new ArrayList<SegmentInfo>();
    private Set<SegmentInfo> segmentSet = new HashSet<SegmentInfo>();
    private transient List<SegmentInfo> cachedUnmodifiableList;
    private transient Set<SegmentInfo> cachedUnmodifiableSet;  
    
    /**
     * If non-null, information about loading segments_N files
     * will be printed here.  @see #setInfoStream.
     */
    private static PrintStream infoStream = null;
  
    public void setFormat(int format) {
      this.format = format;
    }
  
    public int getFormat() {
      return format;
    }
  
    public SegmentInfo info(int i) {
      return segments.get(i);
    }
  
    /**
     * Get the generation (N) of the current segments_N file
     * from a list of files.
     *
     * @param files -- array of file names to check
     */
    public static long getCurrentSegmentGeneration(String[] files) {
      if (files == null) {
        return -1;
      }
      long max = -1;
      for (int i = 0; i < files.length; i++) {
        String file = files[i];
        if (file.startsWith(IndexFileNames.SEGMENTS) && !file.equals(IndexFileNames.SEGMENTS_GEN)) {
          long gen = generationFromSegmentsFileName(file);
          if (gen > max) {
            max = gen;
          }
        }
      }
      return max;
    }
  
    /**
     * Get the generation (N) of the current segments_N file
     * in the directory.
     *
     * @param directory -- directory to search for the latest segments_N file
     */
    public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
      try {
        return getCurrentSegmentGeneration(directory.listAll());
      } catch (NoSuchDirectoryException nsde) {
        return -1;
      }
    }
  
    /**
     * Get the filename of the current segments_N file
     * from a list of files.
     *
     * @param files -- array of file names to check
     */
  
    public static String getCurrentSegmentFileName(String[] files) throws IOException {
      return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                   "",
                                                   getCurrentSegmentGeneration(files));
    }
  
    /**
     * Get the filename of the current segments_N file
     * in the directory.
     *
     * @param directory -- directory to search for the latest segments_N file
     */
    public static String getCurrentSegmentFileName(Directory directory) throws IOException {
      return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                   "",
                                                   getCurrentSegmentGeneration(directory));
    }
  
    /**
     * Get the segments_N filename in use by this segment infos.
     */
    public String getCurrentSegmentFileName() {
      return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                   "",
                                                   lastGeneration);
    }
    
    public String getCurrentSegmentFileNameCRC() {
  	    return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS_CHECKSUM,
  	                                                 "",
  	                                                 lastGeneration);
  	  }
  
    /**
     * Parse the generation off the segments file name and
     * return it.
     */
    public static long generationFromSegmentsFileName(String fileName) {
      if (fileName.equals(IndexFileNames.SEGMENTS)) {
        return 0;
      } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
        return Long.parseLong(fileName.substring(1+IndexFileNames.SEGMENTS.length()),
                              Character.MAX_RADIX);
      } else {
        throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
      }
    }
  
  
    /**
     * Get the next segments_N filename that will be written.
     */
    public String[] getNextSegmentFileName() {
      long nextGeneration;
  
      if (generation == -1) {
        nextGeneration = 1;
      } else {
        nextGeneration = generation+1;
      }
      
      String[] rtn=new String[2];
      rtn[0] =IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                   "",
                                                   nextGeneration);
      rtn[1] =IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS_CHECKSUM,
              "",
              nextGeneration);
      
      return rtn;
    }
  
    /**
     * Read a particular segmentFileName.  Note that this may
     * throw an IOException if a commit is in process.
     *
     * @param directory -- directory containing the segments file
     * @param segmentFileName -- segment file to load
     * @throws CorruptIndexException if the index is corrupt
     * @throws IOException if there is a low-level IO error
     */
    public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException {
      boolean success = false;
  
      // Clear any previous segments:
      this.clear();
  
      String checksumfile=segmentFileName.replaceFirst(IndexFileNames.SEGMENTS, IndexFileNames.SEGMENTS_CHECKSUM);
      
      boolean isusechecksumfile=false;
      long checksum=-1;
      if(directory.fileExists(checksumfile))
      {
          IndexInput chkinput=directory.openInput(checksumfile);
          checksum=chkinput.readLong();
          chkinput.close();
          isusechecksumfile=true;
      }else{
  	System.out.println("can`t found checksum file "+checksumfile);
      }
      
      ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName));
  
      generation = generationFromSegmentsFileName(segmentFileName);
  
      lastGeneration = generation;
  
      try {
        int format = input.readInt();
        // check that it is a format we can understand
        if (format > FORMAT_MINIMUM) {
          throw new IndexFormatTooOldException(input, format,
            FORMAT_MINIMUM, FORMAT_MAXIMUM);
        }
        if (format < FORMAT_MAXIMUM) {
          throw new IndexFormatTooNewException(input, format,
            FORMAT_MINIMUM, FORMAT_MAXIMUM);
        }
        version = input.readLong(); // read version
        counter = input.readInt(); // read counter
        
        for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
          SegmentInfo si = new SegmentInfo(directory, format, input);
          if (si.getVersion() == null) {
            // It's a pre-3.1 segment, upgrade its version to either 3.0 or 2.x
            Directory dir = directory;
            if (si.getDocStoreOffset() != -1) {
              if (si.getDocStoreIsCompoundFile()) {
                dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
                    si.getDocStoreSegment(),
                    IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),IndexFileNames.segmentFileName(
                            si.getDocStoreSegment(),
                            IndexFileNames.COMPOUND_FILE_STORE_EXTENSION_POS), 1024);
              }
            } else if (si.getUseCompoundFile()) {
              dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
                  si.name, IndexFileNames.COMPOUND_FILE_EXTENSION),IndexFileNames.segmentFileName(
                          si.name, IndexFileNames.COMPOUND_FILE_EXTENSION_POS), 1024);
            }
  
            try {
              String store = si.getDocStoreOffset() != -1 ? si.getDocStoreSegment() : si.name;
              si.setVersion(FieldsReader.detectCodeVersion(dir, store));
            } finally {
              // If we opened the directory, close it
              if (dir != directory) dir.close();
            }
          }
          add(si);
        }
        
        if(format >= 0){    // in old format the version number may be at the end of the file
          if (input.getFilePointer() >= input.length())
            version = System.currentTimeMillis(); // old file format without version number
          else
            version = input.readLong(); // read version
        }
  
        if (format <= FORMAT_USER_DATA) {
          if (format <= FORMAT_DIAGNOSTICS) {
            userData = input.readStringStringMap();
          } else if (0 != input.readByte()) {
            userData = Collections.singletonMap("userData", input.readString());
          } else {
            userData = Collections.<String,String>emptyMap();
          }
        } else {
          userData = Collections.<String,String>emptyMap();
        }
  
        if (format <= FORMAT_CHECKSUM) {
          final long checksumNow = input.getChecksum();
          long checksumThen = checksum;
  	if (!isusechecksumfile) {
  	    checksumThen = input.readLong();
  	}
          if (checksumNow != checksumThen)
            throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
        }
        success = true;
      }
      finally {
        input.close();
        if (!success) {
          // Clear any segment infos we had loaded so we
          // have a clean slate on retry:
          this.clear();
        }
      }
    }
  
    /**
     * This version of read uses the retry logic (for lock-less
     * commits) to find the right segments file to load.
     * @throws CorruptIndexException if the index is corrupt
     * @throws IOException if there is a low-level IO error
     */
    public final void read(Directory directory) throws CorruptIndexException, IOException {
  
      generation = lastGeneration = -1;
  
      new FindSegmentsFile(directory) {
  
        @Override
        protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
          read(directory, segmentFileName);
          return null;
        }
      }.run();
    }
  
    // Only non-null after prepareCommit has been called and
    // before finishCommit is called
    ChecksumIndexOutput pendingSegnOutput;
  
    private final void write(Directory directory) throws IOException {
  
      String[] segmentFileName = getNextSegmentFileName();
  
      // Always advance the generation on write:
      if (generation == -1) {
        generation = 1;
      } else {
        generation++;
      }
  
      ChecksumIndexOutput segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName[0]),directory,segmentFileName[1]);
  
      boolean success = false;
  
      try {
        segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
        segnOutput.writeLong(version); 
        segnOutput.writeInt(counter); // write counter
        segnOutput.writeInt(size()); // write infos
        for (SegmentInfo si : this) {
          si.write(segnOutput);
        }
        segnOutput.writeStringStringMap(userData);
        segnOutput.prepareCommit();
        pendingSegnOutput = segnOutput;
        success = true;
      } finally {
        if (!success) {
          // We hit an exception above; try to close the file
          // but suppress any exception:
          IOUtils.closeWhileHandlingException(segnOutput);
          try {
            // Try not to leave a truncated segments_N file in
            // the index:
            directory.deleteFile(segmentFileName[0]);
            directory.deleteFile(segmentFileName[1]);
          } catch (Throwable t) {
            // Suppress so we keep throwing the original exception
          }
        }
      }
    }
  
    /** Prunes any segment whose docs are all deleted. */
    public void pruneDeletedSegments() throws IOException {
      for(final Iterator<SegmentInfo> it = segments.iterator(); it.hasNext();) {
        final SegmentInfo info = it.next();
        if (info.getDelCount() == info.docCount) {
          it.remove();
          segmentSet.remove(info);
        }
      }
      assert segmentSet.size() == segments.size();
    }
  
    /**
     * Returns a copy of this instance, also copying each
     * SegmentInfo.
     */
    
    @Override
    public Object clone() {
      try {
        final SegmentInfos sis = (SegmentInfos) super.clone();
        // deep clone, first recreate all collections:
        sis.segments = new ArrayList<SegmentInfo>(size());
        sis.segmentSet = new HashSet<SegmentInfo>(size());
        sis.cachedUnmodifiableList = null;
        sis.cachedUnmodifiableSet = null;
        for(final SegmentInfo info : this) {
          // dont directly access segments, use add method!!!
          sis.add((SegmentInfo) info.clone());
        }
        sis.userData = new HashMap<String,String>(userData);
        return sis;
      } catch (CloneNotSupportedException e) {
        throw new RuntimeException("should not happen", e);
      }
    }
  
    /**
     * version number when this SegmentInfos was generated.
     */
    public long getVersion() {
      return version;
    }
    public long getGeneration() {
      return generation;
    }
    public long getLastGeneration() {
      return lastGeneration;
    }
  
    /**
     * Current version number from segments file.
     * @throws CorruptIndexException if the index is corrupt
     * @throws IOException if there is a low-level IO error
     */
    public static long readCurrentVersion(Directory directory)
      throws CorruptIndexException, IOException {
  
      // Fully read the segments file: this ensures that it's
      // completely written so that if
      // IndexWriter.prepareCommit has been called (but not
      // yet commit), then the reader will still see itself as
      // current:
      SegmentInfos sis = new SegmentInfos();
      sis.read(directory);
      return sis.version;
    }
  
    /**
     * Returns userData from latest segments file
     * @throws CorruptIndexException if the index is corrupt
     * @throws IOException if there is a low-level IO error
     */
    public static Map<String,String> readCurrentUserData(Directory directory)
      throws CorruptIndexException, IOException {
      SegmentInfos sis = new SegmentInfos();
      sis.read(directory);
      return sis.getUserData();
    }
  
    /** If non-null, information about retries when loading
     * the segments file will be printed to this.
     */
    public static void setInfoStream(PrintStream infoStream) {
      SegmentInfos.infoStream = infoStream;
    }
  
    /* Advanced configuration of retry logic in loading
       segments_N file */
    private static int defaultGenFileRetryCount = 10;
    private static int defaultGenFileRetryPauseMsec = 50;
    private static int defaultGenLookaheadCount = 10;
  
    /**
     * Advanced: set how many times to try loading the
     * segments.gen file contents to determine current segment
     * generation.  This file is only referenced when the
     * primary method (listing the directory) fails.
     */
    public static void setDefaultGenFileRetryCount(int count) {
      defaultGenFileRetryCount = count;
    }
  
    /**
     * @see #setDefaultGenFileRetryCount
     */
    public static int getDefaultGenFileRetryCount() {
      return defaultGenFileRetryCount;
    }
  
    /**
     * Advanced: set how many milliseconds to pause in between
     * attempts to load the segments.gen file.
     */
    public static void setDefaultGenFileRetryPauseMsec(int msec) {
      defaultGenFileRetryPauseMsec = msec;
    }
  
    /**
     * @see #setDefaultGenFileRetryPauseMsec
     */
    public static int getDefaultGenFileRetryPauseMsec() {
      return defaultGenFileRetryPauseMsec;
    }
  
    /**
     * Advanced: set how many times to try incrementing the
     * gen when loading the segments file.  This only runs if
     * the primary (listing directory) and secondary (opening
     * segments.gen file) methods fail to find the segments
     * file.
     */
    public static void setDefaultGenLookaheadCount(int count) {
      defaultGenLookaheadCount = count;
    }
    /**
     * @see #setDefaultGenLookaheadCount
     */
    public static int getDefaultGenLookahedCount() {
      return defaultGenLookaheadCount;
    }
  
    /**
     * @see #setInfoStream
     */
    public static PrintStream getInfoStream() {
      return infoStream;
    }
  
    /**
     * Prints the given message to the infoStream. Note, this method does not
     * check for null infoStream. It assumes this check has been performed by the
     * caller, which is recommended to avoid the (usually) expensive message
     * creation.
     */
    private static void message(String message) {
      infoStream.println("SIS [" + Thread.currentThread().getName() + "]: " + message);
    }
  
    /**
     * Utility class for executing code that needs to do
     * something with the current segments file.  This is
     * necessary with lock-less commits because from the time
     * you locate the current segments file name, until you
     * actually open it, read its contents, or check modified
     * time, etc., it could have been deleted due to a writer
     * commit finishing.
     */
    public abstract static class FindSegmentsFile {
      
      final Directory directory;
  
      public FindSegmentsFile(Directory directory) {
        this.directory = directory;
      }
  
      public Object run() throws CorruptIndexException, IOException {
        return run(null);
      }
      
      public Object run(IndexCommit commit) throws CorruptIndexException, IOException {
        if (commit != null) {
          if (directory != commit.getDirectory())
            throw new IOException("the specified commit does not match the specified Directory");
          return doBody(commit.getSegmentsFileName());
        }
  
        String segmentFileName = null;
        long lastGen = -1;
        long gen = 0;
        int genLookaheadCount = 0;
        IOException exc = null;
        int retryCount = 0;
  
        boolean useFirstMethod = true;
  
        // Loop until we succeed in calling doBody() without
        // hitting an IOException.  An IOException most likely
        // means a commit was in process and has finished, in
        // the time it took us to load the now-old infos files
        // (and segments files).  It's also possible it's a
        // true error (corrupt index).  To distinguish these,
        // on each retry we must see "forward progress" on
        // which generation we are trying to load.  If we
        // don't, then the original error is real and we throw
        // it.
        
        // We have three methods for determining the current
        // generation.  We try the first two in parallel (when
        // useFirstMethod is true), and fall back to the third
        // when necessary.
  
        while(true) {
  
          if (useFirstMethod) {
  
            // List the directory and use the highest
            // segments_N file.  This method works well as long
            // as there is no stale caching on the directory
            // contents (NOTE: NFS clients often have such stale
            // caching):
            String[] files = null;
  
            long genA = -1;
  
            files = directory.listAll();
            
            if (files != null) {
              genA = getCurrentSegmentGeneration(files);
            }
            
            if (infoStream != null) {
              message("directory listing genA=" + genA);
            }
  
            // Also open segments.gen and read its
            // contents.  Then we take the larger of the two
            // gens.  This way, if either approach is hitting
            // a stale cache (NFS) we have a better chance of
            // getting the right generation.
            long genB = -1;
            for(int i=0;i<defaultGenFileRetryCount;i++) {
              IndexInput genInput = null;
              
              if(!directory.fileExists(IndexFileNames.SEGMENTS_GEN))
              {
              	 if (infoStream != null) {
                       message("segments.gen open: FileNotFoundException "+IndexFileNames.SEGMENTS_GEN );
                     }
                     break;
              }
              try {
                genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
              } catch (FileNotFoundException e) {
                if (infoStream != null) {
                  message("segments.gen open: FileNotFoundException " + e);
                }
                break;
              } catch (IOException e) {
                if (infoStream != null) {
                  message("segments.gen open: IOException " + e);
                }
              }
    
              if (genInput != null) {
                try {
                  int version = genInput.readInt();
                  if (version == FORMAT_LOCKLESS) {
                    long gen0 = genInput.readLong();
                    long gen1 = genInput.readLong();
                    if (infoStream != null) {
                      message("fallback check: " + gen0 + "; " + gen1);
                    }
                    if (gen0 == gen1) {
                      // The file is consistent.
                      genB = gen0;
                      break;
                    }
                  }
                } catch (FileNotFoundException err2) {
                    if (infoStream != null) {
                      message("segments.gen open: FileNotFoundException " + err2);
                    }
                    break;
                  
                }
                catch (IOException err2) {
                  // will retry
                } finally {
                  genInput.close();
                }
              }
              try {
                Thread.sleep(defaultGenFileRetryPauseMsec);
              } catch (InterruptedException ie) {
  //              throw new ThreadInterruptedException(ie);
              }
            }
  
            if (infoStream != null) {
              message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
            }
  
            // Pick the larger of the two gen's:
            if (genA > genB)
              gen = genA;
            else
              gen = genB;
  
            if (gen == -1) {
              // Neither approach found a generation
              throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files));
            }
          }
  
          if (useFirstMethod && lastGen == gen && retryCount >= 2) {
            // Give up on first method -- this is 3rd cycle on
            // listing directory and checking gen file to
            // attempt to locate the segments file.
            useFirstMethod = false;
          }
  
          // Second method: since both directory cache and
          // file contents cache seem to be stale, just
          // advance the generation.
          if (!useFirstMethod) {
            if (genLookaheadCount < defaultGenLookaheadCount) {
              gen++;
              genLookaheadCount++;
              if (infoStream != null) {
                message("look ahead increment gen to " + gen);
              }
            } else {
              // All attempts have failed -- throw first exc:
              throw exc;
            }
          } else if (lastGen == gen) {
            // This means we're about to try the same
            // segments_N last tried.
            retryCount++;
          } else {
            // Segment file has advanced since our last loop
            // (we made "progress"), so reset retryCount:
            retryCount = 0;
          }
  
          lastGen = gen;
  
          segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                  "",
                                                                  gen);
  
          try {
            Object v = doBody(segmentFileName);
            if (infoStream != null) {
              message("success on " + segmentFileName);
            }
            return v;
          } catch (IOException err) {
  
            // Save the original root cause:
            if (exc == null) {
              exc = err;
            }
  
            if (infoStream != null) {
              message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retryCount=" + retryCount + "; gen = " + gen);
            }
  
            if (gen > 1 && useFirstMethod && retryCount == 1) {
  
              // This is our second time trying this same segments
              // file (because retryCount is 1), and, there is
              // possibly a segments_(N-1) (because gen > 1).
              // So, check if the segments_(N-1) exists and
              // try it if so:
              String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                                 "",
                                                                                 gen-1);
  
              final boolean prevExists;
              prevExists = directory.fileExists(prevSegmentFileName);
  
              if (prevExists) {
                if (infoStream != null) {
                  message("fallback to prior segment file '" + prevSegmentFileName + "'");
                }
                try {
                  Object v = doBody(prevSegmentFileName);
                  if (infoStream != null) {
                    message("success on fallback " + prevSegmentFileName);
                  }
                  return v;
                } catch (IOException err2) {
                  if (infoStream != null) {
                    message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
                  }
                }
              }
            }
          }
        }
      }
  
      /**
       * Subclass must implement this.  The assumption is an
       * IOException will be thrown if something goes wrong
       * during the processing that could have been caused by
       * a writer committing.
       */
      protected abstract Object doBody(String segmentFileName) throws CorruptIndexException, IOException;
    }
  
    /**
     * Returns a new SegmentInfos containing the SegmentInfo
     * instances in the specified range first (inclusive) to
     * last (exclusive), so total number of segments returned
     * is last-first.
     * @deprecated use {@code asList().subList(first, last)}
     * instead.
     */
    @Deprecated
    public SegmentInfos range(int first, int last) {
      SegmentInfos infos = new SegmentInfos();
      infos.addAll(segments.subList(first, last));
      return infos;
    }
  
    // Carry over generation numbers from another SegmentInfos
    void updateGeneration(SegmentInfos other) {
      lastGeneration = other.lastGeneration;
      generation = other.generation;
    }
  
    final void rollbackCommit(Directory dir) throws IOException {
      if (pendingSegnOutput != null) {
        try {
          pendingSegnOutput.close();
        } catch (Throwable t) {
          // Suppress so we keep throwing the original exception
          // in our caller
        }
  
        // Must carefully compute fileName from "generation"
        // since lastGeneration isn't incremented:
        try {
          final String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                                "",
                                                                               generation);
          final String segmentFileName_crc = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS_CHECKSUM,
                  "",
                 generation);
          dir.deleteFile(segmentFileName);
          dir.deleteFile(segmentFileName_crc);
        } catch (Throwable t) {
          // Suppress so we keep throwing the original exception
          // in our caller
        }
        pendingSegnOutput = null;
      }
    }
  
    /** Call this to start a commit.  This writes the new
     *  segments file, but writes an invalid checksum at the
     *  end, so that it is not visible to readers.  Once this
     *  is called you must call {@link #finishCommit} to complete
     *  the commit or {@link #rollbackCommit} to abort it.
     *  <p>
     *  Note: {@link #changed()} should be called prior to this
     *  method if changes have been made to this {@link SegmentInfos} instance
     *  </p>  
     **/
    final void prepareCommit(Directory dir) throws IOException {
      if (pendingSegnOutput != null)
        throw new IllegalStateException("prepareCommit was already called");
      write(dir);
    }
  
    /** Returns all file names referenced by SegmentInfo
     *  instances matching the provided Directory (ie files
     *  associated with any "external" segments are skipped).
     *  The returned collection is recomputed on each
     *  invocation.  */
    public Collection<String> files(Directory dir, boolean includeSegmentsFile) throws IOException {
      HashSet<String> files = new HashSet<String>();
      if (includeSegmentsFile) {
        files.add(getCurrentSegmentFileName());
        files.add(getCurrentSegmentFileNameCRC());
        
      }
      final int size = size();
      for(int i=0;i<size;i++) {
        final SegmentInfo info = info(i);
        if (info.dir == dir) {
          files.addAll(info(i).files());
        }
      }
      return files;
    }
  
    final void finishCommit(Directory dir) throws IOException {
      if (pendingSegnOutput == null)
        throw new IllegalStateException("prepareCommit was not called");
      boolean success = false;
      try {
        pendingSegnOutput.finishCommit();
        pendingSegnOutput.close();
        pendingSegnOutput = null;
        success = true;
      } finally {
        if (!success)
          rollbackCommit(dir);
      }
  
      // NOTE: if we crash here, we have left a segments_N
      // file in the directory in a possibly corrupt state (if
      // some bytes made it to stable storage and others
      // didn't).  But, the segments_N file includes checksum
      // at the end, which should catch this case.  So when a
      // reader tries to read it, it will throw a
      // CorruptIndexException, which should cause the retry
      // logic in SegmentInfos to kick in and load the last
      // good (previous) segments_N-1 file.
  
      final String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                    "",
                                                                    generation);
      final String fileName_crc = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS_CHECKSUM,
              "",
              generation);
      success = false;
      try {
        dir.sync(Collections.singleton(fileName));
        dir.sync(Collections.singleton(fileName_crc));
        success = true;
      } finally {
        if (!success) {
          try {
            dir.deleteFile(fileName);
            dir.deleteFile(fileName_crc);
          } catch (Throwable t) {
            // Suppress so we keep throwing the original exception
          }
        }
      }
  
      lastGeneration = generation;
  
      try {
        IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
        try {
          genOutput.writeInt(FORMAT_LOCKLESS);
          genOutput.writeLong(generation);
          genOutput.writeLong(generation);
        } finally {
          genOutput.close();
        }
      } catch (ThreadInterruptedException t) {
      	logger.info("sigment error ",t);
        throw t;
      } catch (Throwable t) {
      	logger.info("sigment error ",t);
  
      }
    }
  
    /** Writes & syncs to the Directory dir, taking care to
     *  remove the segments file on exception
     *  <p>
     *  Note: {@link #changed()} should be called prior to this
     *  method if changes have been made to this {@link SegmentInfos} instance
     *  </p>  
     **/
    final void commit(Directory dir) throws IOException {
      prepareCommit(dir);
      finishCommit(dir);
    }
  
    public String toString(Directory directory) {
      StringBuilder buffer = new StringBuilder();
      buffer.append(getCurrentSegmentFileName()).append(": ");
      final int count = size();
      for(int i = 0; i < count; i++) {
        if (i > 0) {
          buffer.append(' ');
        }
        final SegmentInfo info = info(i);
        buffer.append(info.toString(directory, 0));
      }
      return buffer.toString();
    }
  
    public Map<String,String> getUserData() {
      return userData;
    }
  
    void setUserData(Map<String,String> data) {
      if (data == null) {
        userData = Collections.<String,String>emptyMap();
      } else {
        userData = data;
      }
    }
  
    /** Replaces all segments in this instance, but keeps
     *  generation, version, counter so that future commits
     *  remain write once.
     */
    void replace(SegmentInfos other) {
      rollbackSegmentInfos(other.asList());
      lastGeneration = other.lastGeneration;
    }
  
    /** Returns sum of all segment's docCounts.  Note that
     *  this does not include deletions */
    public int totalDocCount() {
      int count = 0;
      for(SegmentInfo info : this) {
        count += info.docCount;
      }
      return count;
    }
  
    /** Call this before committing if changes have been made to the
     *  segments. */
    public void changed() {
      version++;
    }
    
    /** applies all changes caused by committing a merge to this SegmentInfos */
    void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) {
      final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
      boolean inserted = false;
      int newSegIdx = 0;
      for (int segIdx = 0, cnt = segments.size(); segIdx < cnt; segIdx++) {
        assert segIdx >= newSegIdx;
        final SegmentInfo info = segments.get(segIdx);
        if (mergedAway.contains(info)) {
          if (!inserted && !dropSegment) {
            segments.set(segIdx, merge.info);
            inserted = true;
            newSegIdx++;
          }
        } else {
          segments.set(newSegIdx, info);
          newSegIdx++;
        }
      }
  
      // Either we found place to insert segment, or, we did
      // not, but only because all segments we merged became
      // deleted while we are merging, in which case it should
      // be the case that the new segment is also all deleted,
      // we insert it at the beginning if it should not be dropped:
      if (!inserted && !dropSegment) {
        segments.add(0, merge.info);
      }
  
      // the rest of the segments in list are duplicates, so don't remove from map, only list!
      segments.subList(newSegIdx, segments.size()).clear();
      
      // update the Set
      if (!dropSegment) {
        segmentSet.add(merge.info);
      }
      segmentSet.removeAll(mergedAway);
      
      assert segmentSet.size() == segments.size();
    }
  
    List<SegmentInfo> createBackupSegmentInfos(boolean cloneChildren) {
      if (cloneChildren) {
        final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
        for(final SegmentInfo info : this) {
          list.add((SegmentInfo) info.clone());
        }
        return list;
      } else {
        return new ArrayList<SegmentInfo>(segments);
      }
    }
    
    void rollbackSegmentInfos(List<SegmentInfo> infos) {
      this.clear();
      this.addAll(infos);
    }
    
    /** Returns an <b>unmodifiable</b> {@link Iterator} of contained segments in order. */
    // @Override (comment out until Java 6)
    public Iterator<SegmentInfo> iterator() {
      return asList().iterator();
    }
    
    /** Returns all contained segments as an <b>unmodifiable</b> {@link List} view. */
    public List<SegmentInfo> asList() {
      if (cachedUnmodifiableList == null) {
        cachedUnmodifiableList = Collections.unmodifiableList(segments);
      }
      return cachedUnmodifiableList;
    }
    
    /** Returns all contained segments as an <b>unmodifiable</b> {@link Set} view.
     * The iterator is not sorted, use {@link List} view or {@link #iterator} to get all segments in order. */
    public Set<SegmentInfo> asSet() {
      if (cachedUnmodifiableSet == null) {
        cachedUnmodifiableSet = Collections.unmodifiableSet(segmentSet);
      }
      return cachedUnmodifiableSet;
    }
    
    public int size() {
      return segments.size();
    }
  
    public void add(SegmentInfo si) {
      if (segmentSet.contains(si)) {
        throw new IllegalStateException("Cannot add the same segment two times to this SegmentInfos instance");
      }
      segments.add(si);
      segmentSet.add(si);
      assert segmentSet.size() == segments.size();
    }
    
    public void addAll(Iterable<SegmentInfo> sis) {
      for (final SegmentInfo si : sis) {
        this.add(si);
      }
    }
    
    public void clear() {
      segments.clear();
      segmentSet.clear();
    }
    
    public void remove(SegmentInfo si) {
      final int index = this.indexOf(si);
      if (index >= 0) {
        this.remove(index);
      }
    }
    
    public void remove(int index) {
      segmentSet.remove(segments.remove(index));
      assert segmentSet.size() == segments.size();
    }
    
    public boolean contains(SegmentInfo si) {
      return segmentSet.contains(si);
    }
  
    public int indexOf(SegmentInfo si) {
      if (segmentSet.contains(si)) {
        return segments.indexOf(si);
      } else {
        return -1;
      }
    }
  
  }
  