content: /*
   *    This program is free software; you can redistribute it and/or modify
   *    it under the terms of the GNU General Public License as published by
   *    the Free Software Foundation; either version 2 of the License, or
   *    (at your option) any later version.
   *
   *    This program is distributed in the hope that it will be useful,
   *    but WITHOUT ANY WARRANTY; without even the implied warranty of
   *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   *    GNU General Public License for more details.
   *
   *    You should have received a copy of the GNU General Public License
   *    along with this program; if not, write to the Free Software
   *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
  
  /*
   *    ArffLoader.java
   *    Copyright (C) 2000 University of Waikato, Hamilton, New Zealand
   *
   */
  
  package weka.core.converters;
  
  import weka.core.Attribute;
  import weka.core.Instance;
  import weka.core.DenseInstance;
  import weka.core.Instances;
  import weka.core.Option;
  import weka.core.OptionHandler;
  import weka.core.RevisionHandler;
  import weka.core.RevisionUtils;
  import weka.core.SparseInstance;
  import weka.core.Utils;
  
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.io.Reader;
  import java.io.StreamTokenizer;
  import java.io.StringReader;
  import java.net.URL;
  import java.text.ParseException;
  import java.util.ArrayList;
  import java.util.Enumeration;
  import java.util.Vector;
  
  /**
   <!-- globalinfo-start -->
   * Reads a source that is in arff (attribute relation file format) format.
   * <p/>
   <!-- globalinfo-end -->
   *
   * @author Mark Hall (mhall@cs.waikato.ac.nz)
   * @author FracPete (fracpete at waikato dot ac dot nz)
   * @version $Revision: 7792 $
   * @see Loader
   */
  public class ArffLoader 
    extends AbstractFileLoader 
    implements OptionHandler, BatchConverter, IncrementalConverter, 
    URLSourcedLoader {
  
    /** for serialization */
    static final long serialVersionUID = 2726929550544048587L;
    
    /** the file extension */
    public static String FILE_EXTENSION = Instances.FILE_EXTENSION;
    public static String FILE_EXTENSION_COMPRESSED = FILE_EXTENSION + ".gz";
  
    /** the url */
    protected String m_URL = "http://";
  
    /** The reader for the source file. */
    protected transient Reader m_sourceReader = null;
  
    /** The parser for the ARFF file */
    protected transient ArffReader m_ArffReader = null;
    
    /**
     * If false, and reading incrementally, then only one string value (the current
     * one) will be available in the header for each String attribute
     */
    protected boolean m_retainStringValues = false;
    
    /**
     * Reads data from an ARFF file, either in incremental or batch mode. <p/>
     * 
     * Typical code for batch usage:
     * <pre>
     * BufferedReader reader = new BufferedReader(new FileReader("/some/where/file.arff"));
     * ArffReader arff = new ArffReader(reader);
     * Instances data = arff.getData();
     * data.setClassIndex(data.numAttributes() - 1);
     * </pre>
     * 
     * Typical code for incremental usage:
     * <pre>
     * BufferedReader reader = new BufferedReader(new FileReader("/some/where/file.arff"));
     * ArffReader arff = new ArffReader(reader, 1000);
     * Instances data = arff.getStructure();
     * data.setClassIndex(data.numAttributes() - 1);
     * Instance inst;
     * while ((inst = arff.readInstance(data)) != null) {
     *   data.add(inst);
     * }
     * </pre>
     * 
     * @author  Eibe Frank (eibe@cs.waikato.ac.nz)
     * @author  Len Trigg (trigg@cs.waikato.ac.nz)
     * @author  fracpete (fracpete at waikato dot ac dot nz)
     * @version $Revision: 7792 $
     */
    public static class ArffReader
      implements RevisionHandler {
  
      /** the tokenizer for reading the stream */
      protected StreamTokenizer m_Tokenizer;
      
      /** Buffer of values for sparse instance */
      protected double[] m_ValueBuffer;
  
      /** Buffer of indices for sparse instance */
      protected int[] m_IndicesBuffer;
  
      /** the actual data */
      protected Instances m_Data;
  
      /** the number of lines read so far */
      protected int m_Lines;
      
      protected boolean m_batchMode = true;
      
      protected boolean m_retainStringValues = true;
      
      /**
       * Reads the data completely from the reader. The data can be accessed
       * via the <code>getData()</code> method.
       * 
       * @param reader		the reader to use
       * @throws IOException	if something goes wrong
       * @see			#getData()
       */
      public ArffReader(Reader reader) throws IOException {
        m_Tokenizer = new StreamTokenizer(reader);
        initTokenizer();
  
        readHeader(1000);
        initBuffers();
        
        Instance inst;
        while ((inst = readInstance(m_Data)) != null) {
          m_Data.add(inst);
        };
        
        compactify();
      }
      
      public ArffReader(Reader reader, int capacity) throws IOException {
        this(reader, capacity, true, true);
      }
      
      /**
       * Reads only the header and reserves the specified space for instances.
       * Further instances can be read via <code>readInstance()</code>.
       * 
       * @param reader			the reader to use
       * @param capacity 			the capacity of the new dataset 
       * @throws IOException		if something goes wrong
       * @throws IllegalArgumentException	if capacity is negative
       * @see				#getStructure()
       * @see				#readInstance(Instances)
       */
      public ArffReader(Reader reader, int capacity, boolean batch, boolean retainStringVals) 
        throws IOException {
        
        m_batchMode = batch;
        if (batch) {
          m_retainStringValues = true;
        } else {
          m_retainStringValues = retainStringVals;
        }
        
        if (capacity < 0)
  	throw new IllegalArgumentException("Capacity has to be positive!");
  
        m_Tokenizer = new StreamTokenizer(reader);
        initTokenizer();
  
        readHeader(capacity);
        initBuffers();
      }
      
      /**
       * Reads the data without header according to the specified template.
       * The data can be accessed via the <code>getData()</code> method.
       * 
       * @param reader		the reader to use
       * @param template		the template header
       * @param lines		the lines read so far
       * @throws IOException	if something goes wrong
       * @see			#getData()
       */
      public ArffReader(Reader reader, Instances template, int lines) throws IOException {
        this(reader, template, lines, 100);
  
        Instance inst;
        while ((inst = readInstance(m_Data)) != null) {
          m_Data.add(inst);
        };
  
        compactify();
      }
      
      /**
       * Initializes the reader without reading the header according to the 
       * specified template. The data must be read via the 
       * <code>readInstance()</code> method.
       * 
       * @param reader		the reader to use
       * @param template		the template header
       * @param lines		the lines read so far
       * @param capacity 		the capacity of the new dataset 
       * @throws IOException	if something goes wrong
       * @see			#getData()
       */
      public ArffReader(Reader reader, Instances template, int lines, int capacity) throws IOException {
        m_Lines     = lines;
        m_Tokenizer = new StreamTokenizer(reader);
        initTokenizer();
  
        m_Data = new Instances(template, capacity);
        initBuffers();
      }
  
      /**
       * initializes the buffers for sparse instances to be read
       * 
       * @see			#m_ValueBuffer
       * @see			#m_IndicesBuffer
       */
      protected void initBuffers() {
        m_ValueBuffer = new double[m_Data.numAttributes()];
        m_IndicesBuffer = new int[m_Data.numAttributes()];
      }
      
      /**
       * compactifies the data
       */
      protected void compactify() {
        if (m_Data != null)
          m_Data.compactify();
      }
      
      /**
       * Throws error message with line number and last token read.
       *
       * @param msg 		the error message to be thrown
       * @throws IOException 	containing the error message
       */
      protected void errorMessage(String msg) throws IOException {
        String str = msg + ", read " + m_Tokenizer.toString();
        if (m_Lines > 0) {
  	int line = Integer.parseInt(str.replaceAll(".* line ", ""));
  	str = str.replaceAll(" line .*", " line " + (m_Lines + line - 1));
        }
        throw new IOException(str);
      }
  
      /**
       * returns the current line number
       * 
       * @return			the current line number
       */
      public int getLineNo() {
        return m_Lines + m_Tokenizer.lineno();
      }
      
      /**
       * Gets next token, skipping empty lines.
       *
       * @throws IOException 	if reading the next token fails
       */
      protected void getFirstToken() throws IOException {
        while (m_Tokenizer.nextToken() == StreamTokenizer.TT_EOL) {};
        
        if ((m_Tokenizer.ttype == '\'') ||
    	(m_Tokenizer.ttype == '"')) {
          m_Tokenizer.ttype = StreamTokenizer.TT_WORD;
        } else if ((m_Tokenizer.ttype == StreamTokenizer.TT_WORD) &&
    	       (m_Tokenizer.sval.equals("?"))){
          m_Tokenizer.ttype = '?';
        }
      }
  
      /**
       * Gets index, checking for a premature and of line.
       *
       * @throws IOException 	if it finds a premature end of line
       */
      protected void getIndex() throws IOException {
        if (m_Tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
          errorMessage("premature end of line");
        }
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          errorMessage("premature end of file");
        }
      }
      
      /**
       * Gets token and checks if its end of line.
       *
       * @param endOfFileOk 	whether EOF is OK
       * @throws IOException 	if it doesn't find an end of line
       */
      protected void getLastToken(boolean endOfFileOk) throws IOException {
        if ((m_Tokenizer.nextToken() != StreamTokenizer.TT_EOL) &&
    	((m_Tokenizer.ttype != StreamTokenizer.TT_EOF) || !endOfFileOk)) {
          errorMessage("end of line expected");
        }
      }
  
      /**
       * Gets the value of an instance's weight (if one exists)
       *
       * @return the value of the instance's weight, or NaN
       * if no weight has been supplied in the file
       */
      protected double getInstanceWeight() throws IOException {
        double weight = Double.NaN;
        m_Tokenizer.nextToken();
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOL || 
            m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          return weight;
        }
        // see if we can read an instance weight
        //      m_Tokenizer.pushBack();
        if (m_Tokenizer.ttype == '{') {
          m_Tokenizer.nextToken();
          String weightS = m_Tokenizer.sval;
          // try to parse weight as a double
          try {
            weight = Double.parseDouble(weightS);
          } catch (NumberFormatException e) {
            // quietly ignore
            return weight;
          }
          // see if we have the closing brace
          m_Tokenizer.nextToken();
          if (m_Tokenizer.ttype != '}') {
            errorMessage("Problem reading instance weight");
          }
        }
        return weight;
      }
  
      /**
       * Gets next token, checking for a premature and of line.
       *
       * @throws IOException 	if it finds a premature end of line
       */
      protected void getNextToken() throws IOException {
        if (m_Tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
          errorMessage("premature end of line");
        }
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          errorMessage("premature end of file");
        } else if ((m_Tokenizer.ttype == '\'') ||
    	       (m_Tokenizer.ttype == '"')) {
          m_Tokenizer.ttype = StreamTokenizer.TT_WORD;
        } else if ((m_Tokenizer.ttype == StreamTokenizer.TT_WORD) &&
    	       (m_Tokenizer.sval.equals("?"))){
          m_Tokenizer.ttype = '?';
        }
      }
    	
      /**
       * Initializes the StreamTokenizer used for reading the ARFF file.
       */
      protected void initTokenizer(){
        m_Tokenizer.resetSyntax();         
        m_Tokenizer.whitespaceChars(0, ' ');    
        m_Tokenizer.wordChars(' '+1,'\u00FF');
        m_Tokenizer.whitespaceChars(',',',');
        m_Tokenizer.commentChar('%');
        m_Tokenizer.quoteChar('"');
        m_Tokenizer.quoteChar('\'');
        m_Tokenizer.ordinaryChar('{');
        m_Tokenizer.ordinaryChar('}');
        m_Tokenizer.eolIsSignificant(true);
      }
      
      /**
       * Reads a single instance using the tokenizer and returns it. 
       *
       * @param structure 	the dataset header information, will get updated 
       * 				in case of string or relational attributes
       * @return 			null if end of file has been reached
       * @throws IOException 	if the information is not read 
       * successfully
       */ 
      public Instance readInstance(Instances structure) throws IOException {
        return readInstance(structure, true);
      }
      
      /**
       * Reads a single instance using the tokenizer and returns it. 
       *
       * @param structure 	the dataset header information, will get updated 
       * 				in case of string or relational attributes
       * @param flag 		if method should test for carriage return after 
       * 				each instance
       * @return 			null if end of file has been reached
       * @throws IOException 	if the information is not read 
       * successfully
       */ 
      public Instance readInstance(Instances structure, boolean flag) throws IOException {
        return getInstance(structure, flag);
      }
      
      /**
       * Reads a single instance using the tokenizer and returns it. 
       *
       * @param structure 	the dataset header information, will get updated 
       * 				in case of string or relational attributes
       * @param flag 		if method should test for carriage return after 
       * 				each instance
       * @return 			null if end of file has been reached
       * @throws IOException 	if the information is not read 
       * 				successfully
       */ 
      protected Instance getInstance(Instances structure, boolean flag) throws IOException {
        m_Data = structure;
        
        // Check if any attributes have been declared.
        if (m_Data.numAttributes() == 0) {
          errorMessage("no header information available");
        }
  
        // Check if end of file reached.
        getFirstToken();
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          return null;
        }
        
        // Parse instance
        if (m_Tokenizer.ttype == '{') {
          return getInstanceSparse(flag);
        } else {
          return getInstanceFull(flag);
        }
      }
  
      /**
       * Reads a single instance using the tokenizer and returns it.
       *
       * @param flag 		if method should test for carriage return after 
       * 				each instance
       * @return 			null if end of file has been reached
       * @throws IOException 	if the information is not read 
       * 				successfully
       */ 
      protected Instance getInstanceSparse(boolean flag) throws IOException {
        int valIndex, numValues = 0, maxIndex = -1;
        
        // Get values
        do {
          // Get index
          getIndex();
          if (m_Tokenizer.ttype == '}') {
    	break;
          }
     
          // Is index valid?
          try{
    	m_IndicesBuffer[numValues] = Integer.valueOf(m_Tokenizer.sval).intValue();
          } catch (NumberFormatException e) {
    	errorMessage("index number expected");
          }
          if (m_IndicesBuffer[numValues] <= maxIndex) {
    	errorMessage("indices have to be ordered");
          }
          if ((m_IndicesBuffer[numValues] < 0) || 
    	  (m_IndicesBuffer[numValues] >= m_Data.numAttributes())) {
    	errorMessage("index out of bounds");
          }
          maxIndex = m_IndicesBuffer[numValues];
  
          // Get value;
          getNextToken();
  
          // Check if value is missing.
          if  (m_Tokenizer.ttype == '?') {
            m_ValueBuffer[numValues] = Utils.missingValue();
          } else {
  
    	// Check if token is valid.
    	if (m_Tokenizer.ttype != StreamTokenizer.TT_WORD) {
    	  errorMessage("not a valid value");
    	}
            switch (m_Data.attribute(m_IndicesBuffer[numValues]).type()) {
              case Attribute.NOMINAL:
                // Check if value appears in header.
                valIndex = 
                  m_Data.attribute(m_IndicesBuffer[numValues]).indexOfValue(m_Tokenizer.sval);
                if (valIndex == -1) {
                  errorMessage("nominal value not declared in header");
                }
                m_ValueBuffer[numValues] = (double)valIndex;
                break;
    	case Attribute.NUMERIC:
    	  // Check if value is really a number.
    	  try{
    	    m_ValueBuffer[numValues] = Double.valueOf(m_Tokenizer.sval).
    	      doubleValue();
    	  } catch (NumberFormatException e) {
    	    errorMessage("number expected");
    	  }
              break;
    	case Attribute.STRING:
    	  if (m_batchMode || m_retainStringValues) {
    	  m_ValueBuffer[numValues] = 
    	    m_Data.attribute(m_IndicesBuffer[numValues]).addStringValue(m_Tokenizer.sval);
    	  } else {
    	    m_ValueBuffer[numValues] = 0; 
              m_Data.attribute(m_IndicesBuffer[numValues]).setStringValue(m_Tokenizer.sval);
    	  }
              break;
            case Attribute.DATE:
              try {
                m_ValueBuffer[numValues] = 
                  m_Data.attribute(m_IndicesBuffer[numValues]).parseDate(m_Tokenizer.sval);
              } catch (ParseException e) {
                errorMessage("unparseable date: " + m_Tokenizer.sval);
              }
              break;
            case Attribute.RELATIONAL:
              try {
                ArffReader arff = new ArffReader(new StringReader(m_Tokenizer.sval), m_Data.attribute(m_IndicesBuffer[numValues]).relation(), 0);
                Instances data = arff.getData();
                m_ValueBuffer[numValues] = m_Data.attribute(m_IndicesBuffer[numValues]).addRelation(data);
              }
              catch (Exception e) {
                throw new IOException(e.toString() + " of line " + getLineNo());
              }
              break;
            default:
              errorMessage("unknown attribute type in column " + m_IndicesBuffer[numValues]);
    	}
          }
          numValues++;
        } while (true);
  
        double weight = 1.0;
        if (flag) {
          // check for an instance weight
          weight = getInstanceWeight();
          if (!Double.isNaN(weight)) {
            getLastToken(true);
          } else {
            weight = 1.0;
          }        
        }
          
        // Add instance to dataset
        double[] tempValues = new double[numValues];
        int[] tempIndices = new int[numValues];
        System.arraycopy(m_ValueBuffer, 0, tempValues, 0, numValues);
        System.arraycopy(m_IndicesBuffer, 0, tempIndices, 0, numValues);
        Instance inst = new SparseInstance(weight, tempValues, tempIndices, m_Data.numAttributes());
        inst.setDataset(m_Data);
        
        return inst;
      }
  
      /**
       * Reads a single instance using the tokenizer and returns it.
       *
       * @param flag 		if method should test for carriage return after 
       * 				each instance
       * @return 			null if end of file has been reached
       * @throws IOException 	if the information is not read 
       * 				successfully
       */ 
      protected Instance getInstanceFull(boolean flag) throws IOException {
        double[] instance = new double[m_Data.numAttributes()];
        int index;
        
        // Get values for all attributes.
        for (int i = 0; i < m_Data.numAttributes(); i++){
          // Get next token
          if (i > 0) {
    	getNextToken();
          }
                
          // Check if value is missing.
          if  (m_Tokenizer.ttype == '?') {
    	instance[i] = Utils.missingValue();
          } else {
  
    	// Check if token is valid.
    	if (m_Tokenizer.ttype != StreamTokenizer.TT_WORD) {
    	  errorMessage("not a valid value");
    	}
            switch (m_Data.attribute(i).type()) {
            case Attribute.NOMINAL:
    	  // Check if value appears in header.
    	  index = m_Data.attribute(i).indexOfValue(m_Tokenizer.sval);
    	  if (index == -1) {
    	    errorMessage("nominal value not declared in header");
    	  }
    	  instance[i] = (double)index;
              break;
    	case Attribute.NUMERIC:
    	  // Check if value is really a number.
    	  try{
    	    instance[i] = Double.valueOf(m_Tokenizer.sval).
    	      doubleValue();
    	  } catch (NumberFormatException e) {
    	    errorMessage("number expected");
    	  }
              break;
    	case Attribute.STRING:
    	  if (m_batchMode || m_retainStringValues) {
    	    instance[i] = m_Data.attribute(i).addStringValue(m_Tokenizer.sval);
    	  } else {
    	    instance[i] = 0;
    	    m_Data.attribute(i).setStringValue(m_Tokenizer.sval);
    	  }
              break;
            case Attribute.DATE:
              try {
                instance[i] = m_Data.attribute(i).parseDate(m_Tokenizer.sval);
              } catch (ParseException e) {
                errorMessage("unparseable date: " + m_Tokenizer.sval);
              }
              break;
            case Attribute.RELATIONAL:
              try {
                ArffReader arff = new ArffReader(new StringReader(m_Tokenizer.sval), m_Data.attribute(i).relation(), 0);
                Instances data = arff.getData();
                instance[i] = m_Data.attribute(i).addRelation(data);
              }
              catch (Exception e) {
                throw new IOException(e.toString() + " of line " + getLineNo());
              }
              break;
            default:
              errorMessage("unknown attribute type in column " + i);
    	}
          }
        }
        
        double weight = 1.0;
        if (flag) {
          // check for an instance weight
          weight = getInstanceWeight();
          if (!Double.isNaN(weight)) {
            getLastToken(true);
          } else {
            weight = 1.0;
          }
        }
          
        // Add instance to dataset
        Instance inst = new DenseInstance(weight, instance);
        inst.setDataset(m_Data);
        
        return inst;
      }
  
      /**
       * Reads and stores header of an ARFF file.
       *
       * @param capacity 		the number of instances to reserve in the data 
       * 				structure
       * @throws IOException 	if the information is not read 
       * 				successfully
       */ 
      protected void readHeader(int capacity) throws IOException {
        m_Lines = 0;
        String relationName = "";
        
        // Get name of relation.
        getFirstToken();
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          errorMessage("premature end of file");
        }
        if (Instances.ARFF_RELATION.equalsIgnoreCase(m_Tokenizer.sval)) {
          getNextToken();
          relationName = m_Tokenizer.sval;
          getLastToken(false);
        } else {
          errorMessage("keyword " + Instances.ARFF_RELATION + " expected");
        }
  
        // Create vectors to hold information temporarily.
        ArrayList<Attribute> attributes = new ArrayList<Attribute>();
     
        // Get attribute declarations.
        getFirstToken();
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
          errorMessage("premature end of file");
        }
  
        while (Attribute.ARFF_ATTRIBUTE.equalsIgnoreCase(m_Tokenizer.sval)) {
          attributes = parseAttribute(attributes);
        }
  
        // Check if data part follows. We can't easily check for EOL.
        if (!Instances.ARFF_DATA.equalsIgnoreCase(m_Tokenizer.sval)) {
          errorMessage("keyword " + Instances.ARFF_DATA + " expected");
        }
        
        // Check if any attributes have been declared.
        if (attributes.size() == 0) {
          errorMessage("no attributes declared");
        }
        
        m_Data = new Instances(relationName, attributes, capacity);
      }
  
      /**
       * Parses the attribute declaration.
       *
       * @param attributes 		the current attributes vector
       * @return 			the new attributes vector
       * @throws IOException 	if the information is not read 
       * 				successfully
       */
      protected ArrayList<Attribute> parseAttribute(ArrayList<Attribute> attributes) throws IOException {
        String attributeName;
        ArrayList<String> attributeValues;
  
        // Get attribute name.
        getNextToken();
        attributeName = m_Tokenizer.sval;
        getNextToken();
        
        // Check if attribute is nominal.
        if (m_Tokenizer.ttype == StreamTokenizer.TT_WORD) {
          
          // Attribute is real, integer, or string.
          if (m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_REAL) ||
              m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_INTEGER) ||
              m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_NUMERIC)) {
            attributes.add(new Attribute(attributeName, attributes.size()));
            readTillEOL();
          } else if (m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_STRING)) {
            attributes.add(new Attribute(attributeName, (ArrayList<String>)null,
                  attributes.size()));
            readTillEOL();
          } else if (m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_DATE)) {
            String format = null;
            if (m_Tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
              if ((m_Tokenizer.ttype != StreamTokenizer.TT_WORD) &&
                  (m_Tokenizer.ttype != '\'') &&
                  (m_Tokenizer.ttype != '\"')) {
                errorMessage("not a valid date format");
              }
              format = m_Tokenizer.sval;
              readTillEOL();
            } else {
              m_Tokenizer.pushBack();
            }
            attributes.add(new Attribute(attributeName, format, attributes.size()));
            
          } else if (m_Tokenizer.sval.equalsIgnoreCase(Attribute.ARFF_ATTRIBUTE_RELATIONAL)) {
            readTillEOL();
            
            // Read attributes for subrelation
            // First, save current set of attributes
            ArrayList<Attribute> atts = attributes;
            attributes = new ArrayList<Attribute>();
            
            // Now, read attributes until we hit end of declaration of relational value
            getFirstToken();
            if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
              errorMessage("premature end of file");
            }
            do {
              if (Attribute.ARFF_ATTRIBUTE.equalsIgnoreCase(m_Tokenizer.sval)) {
                attributes = parseAttribute(attributes);
              } else if (Attribute.ARFF_END_SUBRELATION.equalsIgnoreCase(m_Tokenizer.sval)) {
                getNextToken();
                if (!attributeName.equalsIgnoreCase(m_Tokenizer.sval)) {
                  errorMessage("declaration of subrelation " + attributeName + 
                        " must be terminated by " + "@end " + attributeName);
                }
                break;
              } else {
                errorMessage("declaration of subrelation " + attributeName + 
                      " must be terminated by " + "@end " + attributeName);
              }
            } while (true);
            
            // Make relation and restore original set of attributes
            Instances relation = new Instances(attributeName, attributes, 0);
            attributes = atts;
            attributes.add(new Attribute(attributeName, relation, attributes.size()));
          } else {
            errorMessage("no valid attribute type or invalid "+
                  "enumeration");
          }
        } else {
          
          // Attribute is nominal.
          attributeValues = new ArrayList<String>();
          m_Tokenizer.pushBack();
          
          // Get values for nominal attribute.
          if (m_Tokenizer.nextToken() != '{') {
            errorMessage("{ expected at beginning of enumeration");
          }
          while (m_Tokenizer.nextToken() != '}') {
            if (m_Tokenizer.ttype == StreamTokenizer.TT_EOL) {
              errorMessage("} expected at end of enumeration");
            } else {
              attributeValues.add(m_Tokenizer.sval);
            }
          }
          attributes.add(new Attribute(attributeName, attributeValues,
                attributes.size()));
        }
        getLastToken(false);
        getFirstToken();
        if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF)
          errorMessage("premature end of file");
        
        return attributes;
      }
  
      /**
       * Reads and skips all tokens before next end of line token.
       *
       * @throws IOException 	in case something goes wrong
       */
      protected void readTillEOL() throws IOException {
        while (m_Tokenizer.nextToken() != StreamTokenizer.TT_EOL) {};
        
        m_Tokenizer.pushBack();
      }
  
      /**
       * Returns the header format
       * 
       * @return			the header format
       */
      public Instances getStructure() {
        return new Instances(m_Data, 0);
      }
      
      /**
       * Returns the data that was read
       * 
       * @return			the data
       */
      public Instances getData() {
        return m_Data;
      }
      
      /**
       * Returns the revision string.
       * 
       * @return		the revision
       */
      public String getRevision() {
        return RevisionUtils.extract("$Revision: 7792 $");
      }
    }
  
    /**
     * Returns a string describing this Loader
     * @return a description of the Loader suitable for
     * displaying in the explorer/experimenter gui
     */
    public String globalInfo() {
      return "Reads a source that is in arff (attribute relation file format) "
        +"format. ";
    }
  
    /**
     * Get the file extension used for arff files
     *
     * @return the file extension
     */
    public String getFileExtension() {
      return FILE_EXTENSION;
    }
  
    /**
     * Gets all the file extensions used for this type of file
     *
     * @return the file extensions
     */
    public String[] getFileExtensions() {
      return new String[]{FILE_EXTENSION, FILE_EXTENSION_COMPRESSED};
    }
  
    /**
     * Returns a description of the file type.
     *
     * @return a short file description
     */
    public String getFileDescription() {
      return "Arff data files";
    }
  
    /**
     * Resets the Loader ready to read a new data set or the
     * same data set again.
     * 
     * @throws IOException if something goes wrong
     */
    public void reset() throws IOException {
      m_structure = null;
      m_ArffReader = null;
      setRetrieval(NONE);
      
      if (m_File != null && !(new File(m_File).isDirectory())) {
        setFile(new File(m_File));
      } else if (m_URL != null && !m_URL.equals("http://")) {
        setURL(m_URL);
      }
    }
  
    /**
     * Resets the Loader object and sets the source of the data set to be 
     * the supplied url.
     *
     * @param url the source url.
     * @throws IOException if an error occurs
     */
    public void setSource(URL url) throws IOException {
      m_structure = null;
      setRetrieval(NONE);
      
      setSource(url.openStream());
  
      m_URL = url.toString();
      // make sure that the file is null so that any calls to
      // reset() work properly
      m_File = null;
    }
    
  
    /**
     * get the File specified as the source
     *
     * @return the source file
     */
    public File retrieveFile() {
      return new File(m_File);
    }
  
    /**
     * sets the source File
     *
     * @param file the source file
     * @throws IOException if an error occurs
     */
    public void setFile(File file) throws IOException {
      m_File = file.getPath();
      setSource(file);
    }
  
    /**
     * Set the url to load from
     *
     * @param url the url to load from
     * @throws IOException if the url can't be set.
     */
    public void setURL(String url) throws IOException {
      m_URL = url;
      setSource(new URL(url));
    }
  
    /**
     * Return the current url
     *
     * @return the current url
     */
    public String retrieveURL() {
      return m_URL;
    }
    
    /**
     * Returns an enumeration describing the available options.
     *
     * @return an enumeration of all the available options.
     */
    public Enumeration listOptions() {
      Vector<Option> result = new Vector<Option>();
      
      result.add(new Option(
          "\tRetain all string attribute values when reading " +
          "incrementally.", "R", 0, "-R"));
      
      return result.elements();
    }
    
    /**
     * Parses a given list of options. <p/>
     *
     <!-- options-start -->
     * Valid options are: <p/>
     * 
     * <pre> -R
     *  Retain all string attribute values when reading incrementally.</pre>
     *  
     <!-- options-end -->
     *
     * @param options the list of options as an array of strings
     * @throws Exception if an option is not supported
     */
    public void setOptions(String[] options) throws Exception {
  
      setRetainStringValues(Utils.getFlag('R', options));
    }
    
    /**
     * Set whether to retain all string values for string in the header
     * when reading incrementally
     * 
     * @param r true if all string values are to be stored (as opposed to
     * just the current one).
     */
    public void setRetainStringValues(boolean r) {
      m_retainStringValues = r;
    }
    
    /**
     * Get whether to retain all string values for string in the header
     * when reading incrementally
     * 
     * @return true if all string values are to be stored (as opposed to
     * just the current one).
     */
    public boolean getRetainStringValues() {
      return m_retainStringValues;
    }
    
    /**
     * the tip text for this property
     * 
     * @return            the tip text
     */
    public String retainStringValuesTipText() {
      return "When reading incrementally, whether to retain all " +
      		"values for string attributes. When set to false " +
      		"only the values for string attributes in the currently " +
      		"read instance will be held in memory.";
    }
    
    /**
     * Gets the current settings of the Classifier.
     *
     * @return an array of strings suitable for passing to setOptions
     */
    public String[] getOptions() {
      Vector<String>      result;
      
      result  = new Vector<String>();
      
      if (getRetainStringValues()) {
        result.add("-R");
      }
      
      return result.toArray(new String[result.size()]);
    }
  
    /**
     * Resets the Loader object and sets the source of the data set to be 
     * the supplied InputStream.
     *
     * @param in the source InputStream.
     * @throws IOException always thrown.
     */
    public void setSource(InputStream in) throws IOException {
      m_File = (new File(System.getProperty("user.dir"))).getAbsolutePath();
      m_URL = "http://";
  
      m_sourceReader = new BufferedReader(new InputStreamReader(in));
    }
  
    /**
     * Determines and returns (if possible) the structure (internally the 
     * header) of the data set as an empty set of instances.
     *
     * @return the structure of the data set as an empty set of Instances
     * @throws IOException if an error occurs
     */
    public Instances getStructure() throws IOException {
  
      if (m_structure == null) {
        if (m_sourceReader == null) {
          throw new IOException("No source has been specified");
        }
        
        try {       
          m_ArffReader = new ArffReader(m_sourceReader, 1, (getRetrieval() == BATCH), 
              m_retainStringValues);       
  	m_structure  = m_ArffReader.getStructure();
        } catch (Exception ex) {
  	throw new IOException("Unable to determine structure as arff (Reason: " + ex.toString() + ").");
        }
      }
  
      return new Instances(m_structure, 0);
    }
  
    /**
     * Return the full data set. If the structure hasn't yet been determined
     * by a call to getStructure then method should do so before processing
     * the rest of the data set.
     *
     * @return the structure of the data set as an empty set of Instances
     * @throws IOException if there is no source or parsing fails
     */
    public Instances getDataSet() throws IOException {
  
      if (m_sourceReader == null) {
        throw new IOException("No source has been specified");
      }
      if (getRetrieval() == INCREMENTAL) {
        throw new IOException("Cannot mix getting Instances in both incremental and batch modes");
      }
      setRetrieval(BATCH);
      if (m_structure == null) {
        getStructure();
      }
  
      // Read all instances
      Instances insts = new Instances(m_structure, 0);
      Instance inst;
      while ((inst = m_ArffReader.readInstance(m_structure)) != null)
        insts.add(inst);
      
      // Instances readIn = new Instances(m_structure);
  
      // close the stream
      m_sourceReader.close();
      
      return insts;
    }
  
    /**
     * Read the data set incrementally---get the next instance in the data 
     * set or returns null if there are no
     * more instances to get. If the structure hasn't yet been 
     * determined by a call to getStructure then method should do so before
     * returning the next instance in the data set.
     *
     * @param structure the dataset header information, will get updated in 
     * case of string or relational attributes
     * @return the next instance in the data set as an Instance object or null
     * if there are no more instances to be read
     * @throws IOException if there is an error during parsing
     */
    public Instance getNextInstance(Instances structure) throws IOException {
  
      m_structure = structure;
  
      if (getRetrieval() == BATCH) {
        throw new IOException("Cannot mix getting Instances in both incremental and batch modes");
      }
      setRetrieval(INCREMENTAL);
  
      Instance current = null;
      if (m_sourceReader != null)
        current = m_ArffReader.readInstance(m_structure);
      
      if ((m_sourceReader != null) && (current == null)) {
        try {
          // close the stream
          m_sourceReader.close();
          m_sourceReader = null;
          //        reset();
        } catch (Exception ex) {
          ex.printStackTrace();
        }
      }
      return current;
    }
    
    /**
     * Returns the revision string.
     * 
     * @return		the revision
     */
    public String getRevision() {
      return RevisionUtils.extract("$Revision: 7792 $");
    }
  
    /**
     * Main method.
     *
     * @param args should contain the name of an input file.
     */
    public static void main(String [] args) {
      runFileLoader(new ArffLoader(), args);
    }
  }
  