content: package org.apache.lucene.index;
  
  /**
   * Copyright 2004 The Apache Software Foundation
   * 
   * Licensed under the Apache License, Version 2.0 (the "License"); you may not
   * use this file except in compliance with the License. You may obtain a copy of
   * the License at
   * 
   * http://www.apache.org/licenses/LICENSE-2.0
   * 
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   * License for the specific language governing permissions and limitations under
   * the License.
   */
  
  import java.io.IOException;
  import java.util.List;
  
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.document.Fieldable;
  import org.apache.lucene.document.NumericField;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.IndexInput;
  import org.apache.lucene.store.IndexOutput;
  import org.apache.lucene.store.RAMDirectory;
  import org.apache.lucene.store.RAMOutputStream;
  import org.apache.lucene.util.IOUtils;
  
  import com.alimama.mdrill.fdtBlockCompress.FdtCompressIndexInput;
  import com.alimama.mdrill.fdtBlockCompress.FdtCompressIndexOutput;
  
  public class FieldsWriter {
      private static final Log LOG = LogFactory.getLog(FieldsWriter.class);
  
    static final int FIELD_IS_TOKENIZED = 1 << 0;
    static final int FIELD_IS_BINARY = 1 << 1;
  
    /** @deprecated Kept for backwards-compatibility with <3.0 indexes; will be removed in 4.0 */
    @Deprecated
    static final int FIELD_IS_COMPRESSED = 1 << 2;
  
    private static final int _NUMERIC_BIT_SHIFT = 3;
    static final int FIELD_IS_NUMERIC_MASK = 0x07 << _NUMERIC_BIT_SHIFT;
  
    static final int FIELD_IS_NUMERIC_INT = 1 << _NUMERIC_BIT_SHIFT;
    static final int FIELD_IS_NUMERIC_LONG = 2 << _NUMERIC_BIT_SHIFT;
    static final int FIELD_IS_NUMERIC_FLOAT = 3 << _NUMERIC_BIT_SHIFT;
    static final int FIELD_IS_NUMERIC_DOUBLE = 4 << _NUMERIC_BIT_SHIFT;
    // currently unused: static final int FIELD_IS_NUMERIC_SHORT = 5 << _NUMERIC_BIT_SHIFT;
    // currently unused: static final int FIELD_IS_NUMERIC_BYTE = 6 << _NUMERIC_BIT_SHIFT;
  
    // the next possible bits are: 1 << 6; 1 << 7
    
    // Original format
    static final int FORMAT = 0;
  
    // Changed strings to UTF8
    static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = 1;
    
    // Lucene 3.0: Removal of compressed fields
    static final int FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS = 2;
  
    // Lucene 3.2: NumericFields are stored in binary format
    static final int FORMAT_LUCENE_3_2_NUMERIC_FIELDS = 3;
  
    // NOTE: if you introduce a new format, make it 1 higher
    // than the current one, and always change this if you
    // switch to a new format!
    static final int FORMAT_CURRENT = FORMAT_LUCENE_3_2_NUMERIC_FIELDS;
    
    private FieldInfos fieldInfos;
  
    // If null - we were supplied with streams, if notnull - we manage them ourselves
    private Directory directory;
    private String segment;
    private IndexOutput fieldsStream;
    private IndexOutput indexStream;
    
    
  
    FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException {
      this.directory = directory;
      this.segment = segment;
      fieldInfos = fn;
  
      boolean success = false;
      try {
  
      	IndexOutput fdt = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
        indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
  
        if(FieldsWriterCompress.isFdtCompress()&&!(directory instanceof RAMDirectory))
        {
            indexStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
            fieldsStream=new FdtCompressIndexOutput(fdt,1024*512);
            fieldsStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
  
        }else{
            fdt.writeInt(FORMAT_CURRENT);
            indexStream.writeInt(FORMAT_CURRENT);
            fieldsStream=fdt;
            fieldsStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
        }
  
        success = true;
      } finally {
        if (!success) {
          abort();
        }
      }
    }
  
    FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) {
      directory = null;
      segment = null;
      fieldInfos = fn;
      fieldsStream = fdt;
      indexStream = fdx;
    }
  
    void setFieldsStream(IndexOutput stream) {
      this.fieldsStream = stream;
    }
  
    // Writes the contents of buffer into the fields stream
    // and adds a new entry for this document into the index
    // stream.  This assumes the buffer was already written
    // in the correct fields format.
    void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException {
  	  long pos=fieldsStream.getFilePointer();
  //	  LOG.info("flushDocument:"+pos+","+numStoredFields);
      indexStream.writeLong(pos);
      fieldsStream.writeVInt(numStoredFields);
      buffer.writeTo(fieldsStream);
    }
  
    void skipDocument() throws IOException {
  	  long pos=fieldsStream.getFilePointer();
  //	  LOG.info("skipDocument:"+pos);
      indexStream.writeLong(pos);
      fieldsStream.writeVInt(0);
    }
  
    void close() throws IOException {
      if (directory != null) {
        try {
          IOUtils.close(fieldsStream, indexStream);
        } finally {
          fieldsStream = indexStream = null;
        }
      }
    }
  
    void abort() {
      if (directory != null) {
        try {
          close();
        } catch (IOException ignored) {
        }
        try {
          directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
        } catch (IOException ignored) {
        }
        try {
          directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
        } catch (IOException ignored) {
        }
      }
    }
  
    final void writeField(FieldInfo fi, Fieldable field) throws IOException {
        fieldsStream.writeVInt(fi.number);
      int bits = 0;
      if (field.isTokenized())
        bits |= FIELD_IS_TOKENIZED;
      if (field.isBinary())
        bits |= FIELD_IS_BINARY;
      if (field instanceof NumericField) {
        switch (((NumericField) field).getDataType()) {
          case INT:
            bits |= FIELD_IS_NUMERIC_INT; break;
          case LONG:
            bits |= FIELD_IS_NUMERIC_LONG; break;
          case FLOAT:
            bits |= FIELD_IS_NUMERIC_FLOAT; break;
          case DOUBLE:
            bits |= FIELD_IS_NUMERIC_DOUBLE; break;
          default:
            assert false : "Should never get here";
        }
      }
      fieldsStream.writeByte((byte) bits);
  
      if (field.isBinary()) {
        final byte[] data;
        final int len;
        final int offset;
        data = field.getBinaryValue();
        len = field.getBinaryLength();
        offset =  field.getBinaryOffset();
  
        fieldsStream.writeVInt(len);
        fieldsStream.writeBytes(data, offset, len);
      } else if (field instanceof NumericField) {
        final NumericField nf = (NumericField) field;
        final Number n = nf.getNumericValue();
        switch (nf.getDataType()) {
          case INT:
              fieldsStream.writeVVInt(n.intValue()); break;
          case LONG:
              fieldsStream.writeVVLong(n.longValue()); break;
          case FLOAT:
              fieldsStream.writeVVVInt(Float.floatToIntBits(n.floatValue())); break;
          case DOUBLE:
              fieldsStream.writeVVVLong(Double.doubleToLongBits(n.doubleValue())); break;
          default:
            assert false : "Should never get here";
        }
      } else {
  	fieldsStream.writeString(field.stringValue());
      }
    }
  
    final void addRawDocuments(final IndexInput stream, long[] lengthsstart,long[] lengthsend, int numDocs) throws IOException {
  	  if(stream instanceof FdtCompressIndexInput)
  	  {
  		  FdtCompressIndexInput inputstream=(FdtCompressIndexInput)stream;
  		    for(int i=0;i<numDocs;i++) {
  			  long position = fieldsStream.getFilePointer();
  		      indexStream.writeLong(position);
  //		      LOG.info("addRawDocuments 1 "+position+","+lengthsstart[i]+","+lengthsend[i]);
  		      inputstream.writeToPos(fieldsStream,lengthsend[i]);
  		    }
  		  return ;
  	  }
  	  
  	  if(fieldsStream instanceof FdtCompressIndexOutput)
  	  {
  		    for(int i=0;i<numDocs;i++) {
  				  long position = fieldsStream.getFilePointer();
  			      indexStream.writeLong(position);
  //			      LOG.info("addRawDocuments 2 "+position+","+lengthsstart[i]+","+lengthsend[i]);
  			      long end=lengthsend[i];
  			      if(end==-1)
  			      {
  			    	  end=stream.length();
  			      }
  				  fieldsStream.copyBytes(stream, end-lengthsstart[i]);
  		    }
  		  
  		  return ;
  	  }
  	  
  	    long position = fieldsStream.getFilePointer();
  	    long start = position;
  	    for(int i=0;i<numDocs;i++) {
  	      indexStream.writeLong(position);
  //	      LOG.info("addRawDocuments 3 "+position+","+lengthsstart[i]+","+lengthsend[i]);
  
  	      long end=lengthsend[i];
  	      if(end==-1)
  	      {
  	    	  end=stream.length();
  	      }
  	      position += end-lengthsstart[i];
  	    }
  	    fieldsStream.copyBytes(stream, position-start);
  	  }
  
    final void addDocument(Document doc) throws IOException {
  	  long pos=fieldsStream.getFilePointer();
      indexStream.writeLong(pos);
  
  //    LOG.info("addDocument "+pos);
  
      int storedCount = 0;
      List<Fieldable> fields = doc.getFields();
      for (Fieldable field : fields) {
        if (field.isStored())
            storedCount++;
      }
      fieldsStream.writeVInt(storedCount);
  
      for (Fieldable field : fields) {
        if (field.isStored())
        {
          writeField(fieldInfos.fieldInfo(field.name()), field);
        }
      }
      
    }
    
  
    
  }
  