content: /*
   *    This program is free software; you can redistribute it and/or modify
   *    it under the terms of the GNU General Public License as published by
   *    the Free Software Foundation; either version 2 of the License, or
   *    (at your option) any later version.
   *
   *    This program is distributed in the hope that it will be useful,
   *    but WITHOUT ANY WARRANTY; without even the implied warranty of
   *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   *    GNU General Public License for more details.
   *
   *    You should have received a copy of the GNU General Public License
   *    along with this program; if not, write to the Free Software
   *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
  
  /*
   *    LogisticBase.java
   *    Copyright (C) 2003 University of Waikato, Hamilton, New Zealand
   *
   */
  
  package weka.classifiers.trees.lmt;
  
  import weka.classifiers.Classifier;
  import weka.classifiers.AbstractClassifier;
  import weka.classifiers.Evaluation;
  import weka.classifiers.functions.SimpleLinearRegression;
  import weka.core.Attribute;
  import weka.core.Instance;
  import weka.core.Instances;
  import weka.core.RevisionUtils;
  import weka.core.Utils;
  import weka.core.WeightedInstancesHandler;
  
  /**
   * Base/helper class for building logistic regression models with the LogitBoost algorithm.
   * Used for building logistic model trees (weka.classifiers.trees.lmt.LMT)
   * and standalone logistic regression (weka.classifiers.functions.SimpleLogistic).
   *
   <!-- options-start -->
   * Valid options are: <p/>
   * 
   * <pre> -D
   *  If set, classifier is run in debug mode and
   *  may output additional info to the console</pre>
   * 
   <!-- options-end -->
   *
   * @author Niels Landwehr
   * @author Marc Sumner
   * @version $Revision: 5928 $
   */
  public class LogisticBase 
      extends AbstractClassifier 
      implements WeightedInstancesHandler {
  
      /** for serialization */
      static final long serialVersionUID = 168765678097825064L;
    
      /** Header-only version of the numeric version of the training data*/
      protected Instances m_numericDataHeader;
      /** 
       * Numeric version of the training data. Original class is replaced by a numeric pseudo-class.
       */
      protected Instances m_numericData;
      
      /** Training data */
      protected Instances m_train;
      
      /** Use cross-validation to determine best number of LogitBoost iterations ?*/
      protected boolean m_useCrossValidation;
  
      /**Use error on probabilities for stopping criterion of LogitBoost? */
      protected boolean m_errorOnProbabilities;
  
      /**Use fixed number of iterations for LogitBoost? (if negative, cross-validate number of iterations)*/
      protected int m_fixedNumIterations;
      
      /**Use heuristic to stop performing LogitBoost iterations earlier?
       * If enabled, LogitBoost is stopped if the current (local) minimum of the error on a test set as 
       * a function of the number of iterations has not changed for m_heuristicStop iterations.
       */
      protected int m_heuristicStop = 50;
   
      /**The number of LogitBoost iterations performed.*/
      protected int m_numRegressions = 0;
      
      /**The maximum number of LogitBoost iterations*/
      protected int m_maxIterations;
      
      /**The number of different classes*/
      protected int m_numClasses;
  
      /**Array holding the simple regression functions fit by LogitBoost*/
      protected SimpleLinearRegression[][] m_regressions;
          
      /**Number of folds for cross-validating number of LogitBoost iterations*/
      protected static int m_numFoldsBoosting = 5;
  
      /**Threshold on the Z-value for LogitBoost*/
      protected static final double Z_MAX = 3;
      
      /** If true, the AIC is used to choose the best iteration*/
      private boolean m_useAIC = false;
      
      /** Effective number of parameters used for AIC / BIC automatic stopping */
      protected double m_numParameters = 0;
      
      /**Threshold for trimming weights. Instances with a weight lower than this (as a percentage
       * of total weights) are not included in the regression fit.
       **/
      protected double m_weightTrimBeta = 0;
  
      /**
       * Constructor that creates LogisticBase object with standard options.
       */
      public LogisticBase(){
  	m_fixedNumIterations = -1;
  	m_useCrossValidation = true;
  	m_errorOnProbabilities = false;	
  	m_maxIterations = 500;
          m_useAIC = false;
          m_numParameters = 0;
      }
      
      /**
       * Constructor to create LogisticBase object.
       * @param numBoostingIterations fixed number of iterations for LogitBoost (if negative, use cross-validation or
       * stopping criterion on the training data).
       * @param useCrossValidation cross-validate number of LogitBoost iterations (if false, use stopping 
       * criterion on the training data).
       * @param errorOnProbabilities if true, use error on probabilities 
       * instead of misclassification for stopping criterion of LogitBoost
       */
      public LogisticBase(int numBoostingIterations, boolean useCrossValidation, boolean errorOnProbabilities){
  	m_fixedNumIterations = numBoostingIterations;
  	m_useCrossValidation = useCrossValidation;
  	m_errorOnProbabilities = errorOnProbabilities;	
  	m_maxIterations = 500;
          m_useAIC = false;
          m_numParameters = 0;
      }    
  
      /**
       * Builds the logistic regression model usiing LogitBoost.
       * 
       * @param data the training data
       * @throws Exception if something goes wrong
       */
      public void buildClassifier(Instances data) throws Exception {			
  
  	m_train = new Instances(data);
  	
  	m_numClasses = m_train.numClasses();
  	
  	//init the array of simple regression functions
  	m_regressions = initRegressions();
  	m_numRegressions = 0;
  
  	//get numeric version of the training data (class variable replaced  by numeric pseudo-class)
  	m_numericData = getNumericData(m_train);	
  	
  	//save header info
  	m_numericDataHeader = new Instances(m_numericData, 0);
  	
  	
  	if (m_fixedNumIterations > 0) {
  	    //run LogitBoost for fixed number of iterations
  	    performBoosting(m_fixedNumIterations);
  	} else if (m_useAIC) { // Marc had this after the test for m_useCrossValidation. Changed by Eibe.
              //run LogitBoost using information criterion for stopping
              performBoostingInfCriterion();
          } else if (m_useCrossValidation) {
  	    //cross-validate number of LogitBoost iterations
  	    performBoostingCV();
  	} else {
  	    //run LogitBoost with number of iterations that minimizes error on the training set
  	    performBoosting();
  	}	
  	
  	//only keep the simple regression functions that correspond to the selected number of LogitBoost iterations
  	m_regressions = selectRegressions(m_regressions);	
      }   
  
      /**
       * Runs LogitBoost, determining the best number of iterations by cross-validation.
       * 
       * @throws Exception if something goes wrong
       */
      protected void performBoostingCV() throws Exception{			
  	
  	//completed iteration keeps track of the number of iterations that have been
  	//performed in every fold (some might stop earlier than others). 
  	//Best iteration is selected only from these.
  	int completedIterations = m_maxIterations;
  	
  	Instances allData = new Instances(m_train);
  	
  	allData.stratify(m_numFoldsBoosting);	      
  
  	double[] error = new double[m_maxIterations + 1];	
  	
  	for (int i = 0; i < m_numFoldsBoosting; i++) {
  	    //split into training/test data in fold
  	    Instances train = allData.trainCV(m_numFoldsBoosting,i);
  	    Instances test = allData.testCV(m_numFoldsBoosting,i);
  
  	    //initialize LogitBoost
  	    m_numRegressions = 0;
  	    m_regressions = initRegressions();
  
  	    //run LogitBoost iterations
  	    int iterations = performBoosting(train,test,error,completedIterations);	    
  	    if (iterations < completedIterations) completedIterations = iterations;	    
  	}
  	
  	//determine iteration with minimum error over the folds
  	int bestIteration = getBestIteration(error,completedIterations);
  	
  	//rebuild model on all of the training data
  	m_numRegressions = 0;
  	performBoosting(bestIteration);
      }    
      
      /**
       * Runs LogitBoost, determining the best number of iterations by an information criterion (currently AIC).
       */
      protected void performBoostingInfCriterion() throws Exception{
          
          double criterion = 0.0;
          double bestCriterion = Double.MAX_VALUE;
          int bestIteration = 0;
          int noMin = 0;
          
          // Variable to keep track of criterion values (AIC)
          double criterionValue = Double.MAX_VALUE;
          
          // initialize Ys/Fs/ps
          double[][] trainYs = getYs(m_train);
          double[][] trainFs = getFs(m_numericData);
          double[][] probs = getProbs(trainFs);
          
          // Array with true/false if the attribute is included in the model or not
          boolean[][] attributes = new boolean[m_numClasses][m_numericDataHeader.numAttributes()];
          
          int iteration = 0;
          while (iteration < m_maxIterations) {
              
              //perform single LogitBoost iteration
              boolean foundAttribute = performIteration(iteration, trainYs, trainFs, probs, m_numericData);
              if (foundAttribute) {
                  iteration++;
                  m_numRegressions = iteration;
              } else {
                  //could not fit simple linear regression: stop LogitBoost
                  break;
              }
              
              double numberOfAttributes = m_numParameters + iteration;
              
              // Fill criterion array values
              criterionValue = 2.0 * negativeLogLikelihood(trainYs, probs) +
                2.0 * numberOfAttributes;
  
              //heuristic: stop LogitBoost if the current minimum has not changed for <m_heuristicStop> iterations
              if (noMin > m_heuristicStop) break;
              if (criterionValue < bestCriterion) {
                  bestCriterion = criterionValue;
                  bestIteration = iteration;
                  noMin = 0;
              } else {
                  noMin++;
              }
          }
  
          m_numRegressions = 0;
          performBoosting(bestIteration);
      }
  
      /**
       * Runs LogitBoost on a training set and monitors the error on a test set.
       * Used for running one fold when cross-validating the number of LogitBoost iterations.
       * @param train the training set
       * @param test the test set
       * @param error array to hold the logged error values
       * @param maxIterations the maximum number of LogitBoost iterations to run
       * @return the number of completed LogitBoost iterations (can be smaller than maxIterations 
       * if the heuristic for early stopping is active or there is a problem while fitting the regressions 
       * in LogitBoost).
       * @throws Exception if something goes wrong
       */
      protected int performBoosting(Instances train, Instances test, 
  				  double[] error, int maxIterations) throws Exception{
  	
  	//get numeric version of the (sub)set of training instances
  	Instances numericTrain = getNumericData(train);		
  
  	//initialize Ys/Fs/ps 
  	double[][] trainYs = getYs(train);
  	double[][] trainFs = getFs(numericTrain);		
  	double[][] probs = getProbs(trainFs);	
  
  	int iteration = 0;
  
   	int noMin = 0;
  	double lastMin = Double.MAX_VALUE;	
  	
  	if (m_errorOnProbabilities) error[0] += getMeanAbsoluteError(test);
  	else error[0] += getErrorRate(test);
  	
  	while (iteration < maxIterations) {
  	  
  	    //perform single LogitBoost iteration
  	    boolean foundAttribute = performIteration(iteration, trainYs, trainFs, probs, numericTrain);
  	    if (foundAttribute) {
  		iteration++;
  		m_numRegressions = iteration;
  	    } else {
  		//could not fit simple linear regression: stop LogitBoost
  		break;
  	    }
  	    
  	    if (m_errorOnProbabilities) error[iteration] += getMeanAbsoluteError(test);
  	    else error[iteration] += getErrorRate(test);
  	  
  	    //heuristic: stop LogitBoost if the current minimum has not changed for <m_heuristicStop> iterations
  	    if (noMin > m_heuristicStop) break;
  	    if (error[iteration] < lastMin) {
  		lastMin = error[iteration];
  		noMin = 0;
  	    } else {
  		noMin++;
  	    }	    	      	    
  	}
  
  	return iteration;
      }
  
      /**
       * Runs LogitBoost with a fixed number of iterations.
       * @param numIterations the number of iterations to run
       * @throws Exception if something goes wrong
       */
      protected void performBoosting(int numIterations) throws Exception{
  
  	//initialize Ys/Fs/ps 
  	double[][] trainYs = getYs(m_train);
  	double[][] trainFs = getFs(m_numericData);		
  	double[][] probs = getProbs(trainFs);
  	
  	int iteration = 0;
  
  	//run iterations
  	while (iteration < numIterations) {
  	    boolean foundAttribute = performIteration(iteration, trainYs, trainFs, probs, m_numericData);
  	    if (foundAttribute) iteration++;
  	    else break;
  	}
  	
  	m_numRegressions = iteration;
      }
      
      /**
       * Runs LogitBoost using the stopping criterion on the training set.
       * The number of iterations is used that gives the lowest error on the training set, either misclassification
       * or error on probabilities (depending on the errorOnProbabilities option).
       * @throws Exception if something goes wrong
       */
      protected void performBoosting() throws Exception{
  	
  	//initialize Ys/Fs/ps
  	double[][] trainYs = getYs(m_train);
  	double[][] trainFs = getFs(m_numericData);		
  	double[][] probs = getProbs(trainFs);	
  
  	int iteration = 0;
  
  	double[] trainErrors = new double[m_maxIterations+1];
  	trainErrors[0] = getErrorRate(m_train);
  	
  	int noMin = 0;
  	double lastMin = Double.MAX_VALUE;
  	
  	while (iteration < m_maxIterations) {
  	    boolean foundAttribute = performIteration(iteration, trainYs, trainFs, probs, m_numericData);
  	    if (foundAttribute) {
  		iteration++;
  		m_numRegressions = iteration;
  	    } else {		
  		//could not fit simple regression
  		break;
  	    }
  	    
  	    trainErrors[iteration] = getErrorRate(m_train);	    
  	 
  	    //heuristic: stop LogitBoost if the current minimum has not changed for <m_heuristicStop> iterations
  	    if (noMin > m_heuristicStop) break;	    
  	    if (trainErrors[iteration] < lastMin) {
  		lastMin = trainErrors[iteration];
  		noMin = 0;
  	    } else {
  		noMin++;
  	    }
  	}
  	
  	//find iteration with best error
          m_numRegressions = getBestIteration(trainErrors, iteration);	
      }
  
      /**
       * Returns the misclassification error of the current model on a set of instances.
       * @param data the set of instances
       * @return the error rate
       * @throws Exception if something goes wrong
       */
      protected double getErrorRate(Instances data) throws Exception {
  	Evaluation eval = new Evaluation(data);
  	eval.evaluateModel(this,data);
  	return eval.errorRate();
      }
  
      /**
       * Returns the error of the probability estimates for the current model on a set of instances.
       * @param data the set of instances
       * @return the error
       * @throws Exception if something goes wrong
       */
      protected double getMeanAbsoluteError(Instances data) throws Exception {
  	Evaluation eval = new Evaluation(data);
  	eval.evaluateModel(this,data);
  	return eval.meanAbsoluteError();
      }
  
      /**
       * Helper function to find the minimum in an array of error values.
       * 
       * @param errors an array containing errors
       * @param maxIteration the maximum of iterations
       * @return the minimum
       */
      protected int getBestIteration(double[] errors, int maxIteration) {
  	double bestError = errors[0];
  	int bestIteration = 0;
  	for (int i = 1; i <= maxIteration; i++) {	    
  	    if (errors[i] < bestError) {
  		bestError = errors[i];
  		bestIteration = i;		
  	    }
  	} 
  	return bestIteration;
      }
  
      /**
       * Performs a single iteration of LogitBoost, and updates the model accordingly.
       * A simple regression function is fit to the response and added to the m_regressions array.
       * @param iteration the current iteration 
       * @param trainYs the y-values (see description of LogitBoost) for the model trained so far
       * @param trainFs the F-values (see description of LogitBoost) for the model trained so far
       * @param probs the p-values (see description of LogitBoost) for the model trained so far
       * @param trainNumeric numeric version of the training data
       * @return returns true if iteration performed successfully, false if no simple regression function 
       * could be fitted.
       * @throws Exception if something goes wrong
       */
      protected boolean performIteration(int iteration, 
  				       double[][] trainYs,
  				       double[][] trainFs,
  				       double[][] probs,
  				       Instances trainNumeric) throws Exception {
  	
  	for (int j = 0; j < m_numClasses; j++) {
              // Keep track of sum of weights
              double[] weights = new double[trainNumeric.numInstances()];
              double weightSum = 0.0;
  	    
  	    //make copy of data (need to save the weights) 
  	    Instances boostData = new Instances(trainNumeric);
  	    
  	    for (int i = 0; i < trainNumeric.numInstances(); i++) {
  		
  		//compute response and weight
  		double p = probs[i][j];
  		double actual = trainYs[i][j];
  		double z = getZ(actual, p);
  		double w = (actual - p) / z;
  		
  		//set values for instance 
  		Instance current = boostData.instance(i);
  		current.setValue(boostData.classIndex(), z);
  		current.setWeight(current.weight() * w);				
                  
                  weights[i] = current.weight();
                  weightSum += current.weight();
  	    }
              
              Instances instancesCopy = new Instances(boostData);
              
              if (weightSum > 0) {
                  // Only the (1-beta)th quantile of instances are sent to the base classifier
                  if (m_weightTrimBeta > 0) {
                      double weightPercentage = 0.0;
                      int[] weightsOrder = new int[trainNumeric.numInstances()];
                      weightsOrder = Utils.sort(weights);
                      instancesCopy.delete();
                      
                      
                      for (int i = weightsOrder.length-1; (i >= 0) && (weightPercentage < (1-m_weightTrimBeta)); i--) {
                          instancesCopy.add(boostData.instance(weightsOrder[i]));
                          weightPercentage += (weights[weightsOrder[i]] / weightSum);
                          
                      }
                  }
                  
                  //Scale the weights
                  weightSum = instancesCopy.sumOfWeights();
                  for (int i = 0; i < instancesCopy.numInstances(); i++) {
                      Instance current = instancesCopy.instance(i);
                      current.setWeight(current.weight() * (double)instancesCopy.numInstances() / weightSum);
                  }
              }
  	    
  	    //fit simple regression function
  	    m_regressions[j][iteration].buildClassifier(instancesCopy);
  	    
  	    boolean foundAttribute = m_regressions[j][iteration].foundUsefulAttribute();
  	    if (!foundAttribute) {
  		//could not fit simple regression function
  		return false;
  	    }
  	    
  	}
  	
  	// Evaluate / increment trainFs from the classifier
  	for (int i = 0; i < trainFs.length; i++) {
  	    double [] pred = new double [m_numClasses];
  	    double predSum = 0;
  	    for (int j = 0; j < m_numClasses; j++) {
  		pred[j] = m_regressions[j][iteration]
  		    .classifyInstance(trainNumeric.instance(i));
  		predSum += pred[j];
  	    }
  	    predSum /= m_numClasses;
  	    for (int j = 0; j < m_numClasses; j++) {
  		trainFs[i][j] += (pred[j] - predSum) * (m_numClasses - 1) 
  		    / m_numClasses;
  	    }
  	}
  	
  	// Compute the current probability estimates
  	for (int i = 0; i < trainYs.length; i++) {
  	    probs[i] = probs(trainFs[i]);
  	}
  	return true;
      }    
  
      /**
       * Helper function to initialize m_regressions.
       * 
       * @return the generated classifiers
       */
      protected SimpleLinearRegression[][] initRegressions(){
  	SimpleLinearRegression[][] classifiers =   
  	    new SimpleLinearRegression[m_numClasses][m_maxIterations];
  	for (int j = 0; j < m_numClasses; j++) {
  	    for (int i = 0; i < m_maxIterations; i++) {
  		classifiers[j][i] = new SimpleLinearRegression();
  		classifiers[j][i].setSuppressErrorMessage(true);
  	    }
  	}
  	return classifiers;
      }
  
      /**
       * Converts training data to numeric version. The class variable is replaced by a pseudo-class 
       * used by LogitBoost.
       * 
       * @param data the data to convert
       * @return the converted data
       * @throws Exception if something goes wrong
       */
      protected Instances getNumericData(Instances data) throws Exception{
  	Instances numericData = new Instances(data);
  	
  	int classIndex = numericData.classIndex();
  	numericData.setClassIndex(-1);
  	numericData.deleteAttributeAt(classIndex);
  	numericData.insertAttributeAt(new Attribute("'pseudo class'"), classIndex);
  	numericData.setClassIndex(classIndex);
  	return numericData;
      }
      
      /**
       * Helper function for cutting back m_regressions to the set of classifiers 
       * (corresponsing to the number of LogitBoost iterations) that gave the 
       * smallest error.
       * 
       * @param classifiers the original set of classifiers
       * @return the cut back set of classifiers
       */
      protected SimpleLinearRegression[][] selectRegressions(SimpleLinearRegression[][] classifiers){
  	SimpleLinearRegression[][] goodClassifiers = 
  	    new SimpleLinearRegression[m_numClasses][m_numRegressions];
  	
  	for (int j = 0; j < m_numClasses; j++) {
  	    for (int i = 0; i < m_numRegressions; i++) {
  		goodClassifiers[j][i] = classifiers[j][i];
  	    }
  	}
  	return goodClassifiers;
      }		
      
      /**
       * Computes the LogitBoost response variable from y/p values 
       * (actual/estimated class probabilities).
       * 
       * @param actual the actual class probability
       * @param p the estimated class probability
       * @return the LogitBoost response
       */
      protected double getZ(double actual, double p) {
  	double z;
  	if (actual == 1) {
  	    z = 1.0 / p;
  	    if (z > Z_MAX) { // threshold
  		z = Z_MAX;
  	    }
  	} else {
  	    z = -1.0 / (1.0 - p);
  	    if (z < -Z_MAX) { // threshold
  		z = -Z_MAX;
  	    }
  	}
  	return z;
      }
      
      /**
       * Computes the LogitBoost response for an array of y/p values 
       * (actual/estimated class probabilities).
       * 
       * @param dataYs the actual class probabilities
       * @param probs the estimated class probabilities
       * @return the LogitBoost response
       */
      protected double[][] getZs(double[][] probs, double[][] dataYs) {
  	
  	double[][] dataZs = new double[probs.length][m_numClasses];
  	for (int j = 0; j < m_numClasses; j++) 
  	    for (int i = 0; i < probs.length; i++) dataZs[i][j] = getZ(dataYs[i][j], probs[i][j]);
  	return dataZs;
      }
      
      /**
       * Computes the LogitBoost weights from an array of y/p values 
       * (actual/estimated class probabilities).
       * 
       * @param dataYs the actual class probabilities
       * @param probs the estimated class probabilities
       * @return the LogitBoost weights
       */
      protected double[][] getWs(double[][] probs, double[][] dataYs) {
  	
  	double[][] dataWs = new double[probs.length][m_numClasses];
  	for (int j = 0; j < m_numClasses; j++) 
  	    for (int i = 0; i < probs.length; i++){
  	    double z = getZ(dataYs[i][j], probs[i][j]);
  	    dataWs[i][j] = (dataYs[i][j] - probs[i][j]) / z;
  	    }
  	return dataWs;
      }
  
      /**
       * Computes the p-values (probabilities for the classes) from the F-values 
       * of the logistic model.
       * 
       * @param Fs the F-values
       * @return the p-values
       */
      protected double[] probs(double[] Fs) {
  	
  	double maxF = -Double.MAX_VALUE;
  	for (int i = 0; i < Fs.length; i++) {
  	    if (Fs[i] > maxF) {
  		maxF = Fs[i];
  	    }
  	}   
  	double sum = 0;
  	double[] probs = new double[Fs.length];
  	for (int i = 0; i < Fs.length; i++) {
  	    probs[i] = Math.exp(Fs[i] - maxF);   
  	    sum += probs[i];
  	}
  	
  	Utils.normalize(probs, sum);
  	return probs;
      }
  
      /**
       * Computes the Y-values (actual class probabilities) for a set of instances.
       * 
       * @param data the data to compute the Y-values from
       * @return the Y-values
       */
      protected double[][] getYs(Instances data){
  	
  	double [][] dataYs = new double [data.numInstances()][m_numClasses];
  	for (int j = 0; j < m_numClasses; j++) {
  	    for (int k = 0; k < data.numInstances(); k++) {
  		dataYs[k][j] = (data.instance(k).classValue() == j) ? 
  		    1.0: 0.0;
  	    }
  	}
  	return dataYs;
      }
  
      /**
       * Computes the F-values for a single instance.
       * 
       * @param instance the instance to compute the F-values for
       * @return the F-values
       * @throws Exception if something goes wrong
       */
      protected double[] getFs(Instance instance) throws Exception{
  	
  	double [] pred = new double [m_numClasses];
  	double [] instanceFs = new double [m_numClasses]; 
  	
  	//add up the predictions from the simple regression functions
  	for (int i = 0; i < m_numRegressions; i++) {
  	    double predSum = 0;
  	    for (int j = 0; j < m_numClasses; j++) {
  		pred[j] = m_regressions[j][i].classifyInstance(instance);
  		predSum += pred[j];
  	    }
  	    predSum /= m_numClasses;
  	    for (int j = 0; j < m_numClasses; j++) {
  		instanceFs[j] += (pred[j] - predSum) * (m_numClasses - 1) 
  		    / m_numClasses;
  	    }
  	}	
  	
  	return instanceFs; 
      } 
      
      /**
       * Computes the F-values for a set of instances.
       * 
       * @param data the data to work on
       * @return the F-values
       * @throws Exception if something goes wrong
       */
      protected double[][] getFs(Instances data) throws Exception{
  	
  	double[][] dataFs = new double[data.numInstances()][];
         
  	for (int k = 0; k < data.numInstances(); k++) {
  	    dataFs[k] = getFs(data.instance(k));
  	}
  	
  	return dataFs;	
      }   
  
      /**
       * Computes the p-values (probabilities for the different classes) from 
       * the F-values for a set of instances.
       * 
       * @param dataFs the F-values
       * @return the p-values
       */
      protected double[][] getProbs(double[][] dataFs){
  	
  	int numInstances = dataFs.length;
  	double[][] probs = new double[numInstances][];
  	
  	for (int k = 0; k < numInstances; k++) {       
  	    probs[k] = probs(dataFs[k]);
  	}
  	return probs;
      }
      
      /**
       * Returns the negative loglikelihood of the Y-values (actual class probabilities) given the 
       * p-values (current probability estimates).
       * 
       * @param dataYs the Y-values
       * @param probs the p-values
       * @return the likelihood
       */
      protected double negativeLogLikelihood(double[][] dataYs, double[][] probs) {
  	
  	double logLikelihood = 0;
  	for (int i = 0; i < dataYs.length; i++) {
  	    for (int j = 0; j < m_numClasses; j++) {
  		if (dataYs[i][j] == 1.0) {
  		    logLikelihood -= Math.log(probs[i][j]);
  		}
  	    }
  	}
  	return logLikelihood;// / (double)dataYs.length;
      }
  
      /**
       * Returns an array of the indices of the attributes used in the logistic model.
       * The first dimension is the class, the second dimension holds a list of attribute indices.
       * Attribute indices start at zero.
       * @return the array of attribute indices
       */
      public int[][] getUsedAttributes(){
  	
  	int[][] usedAttributes = new int[m_numClasses][];
  	
  	//first extract coefficients
  	double[][] coefficients = getCoefficients();
  	
  	for (int j = 0; j < m_numClasses; j++){
  	    
  	    //boolean array indicating if attribute used
  	    boolean[] attributes = new boolean[m_numericDataHeader.numAttributes()];
  	    for (int i = 0; i < attributes.length; i++) {
  		//attribute used if coefficient > 0
  		if (!Utils.eq(coefficients[j][i + 1],0)) attributes[i] = true;
  	    }
  	    	    
  	    int numAttributes = 0;
  	    for (int i = 0; i < m_numericDataHeader.numAttributes(); i++) if (attributes[i]) numAttributes++;
  	    
  	    //"collect" all attributes into array of indices
  	    int[] usedAttributesClass = new int[numAttributes];
  	    int count = 0;
  	    for (int i = 0; i < m_numericDataHeader.numAttributes(); i++) {
  		if (attributes[i]) {
  		usedAttributesClass[count] = i;
  		count++;
  		} 
  	    }
  	    
  	    usedAttributes[j] = usedAttributesClass;
  	}
  	
  	return usedAttributes;
      }
  
      /**
       * The number of LogitBoost iterations performed (= the number of simple 
       * regression functions fit).
       * 
       * @return the number of LogitBoost iterations performed 
       */
      public int getNumRegressions() {
  	return m_numRegressions;
      }
      
      /**
       * Get the value of weightTrimBeta.
       *
       * @return Value of weightTrimBeta.
       */
      public double getWeightTrimBeta(){
          return m_weightTrimBeta;
      }
      
      /**
       * Get the value of useAIC.
       *
       * @return Value of useAIC.
       */
      public boolean getUseAIC(){
          return m_useAIC;
      }
  
      /**
       * Sets the parameter "maxIterations".
       * 
       * @param maxIterations the maximum iterations
       */
      public void setMaxIterations(int maxIterations) {
  	m_maxIterations = maxIterations;
      }
      
      /**
       * Sets the option "heuristicStop".
       * 
       * @param heuristicStop the heuristic stop to use
       */
      public void setHeuristicStop(int heuristicStop){
  	m_heuristicStop = heuristicStop;
      }
      
      /**
       * Sets the option "weightTrimBeta".
       */
      public void setWeightTrimBeta(double w){
          m_weightTrimBeta = w;
      }
      
      /**
       * Set the value of useAIC.
       *
       * @param c Value to assign to useAIC.
       */
      public void setUseAIC(boolean c){
          m_useAIC = c;
      }
  
      /**
       * Returns the maxIterations parameter.
       * 
       * @return the maximum iteration
       */
      public int getMaxIterations(){
  	return m_maxIterations;
      }
          
      /**
       * Returns an array holding the coefficients of the logistic model.
       * First dimension is the class, the second one holds a list of coefficients.
       * At position zero, the constant term of the model is stored, then, the coefficients for
       * the attributes in ascending order.
       * @return the array of coefficients
       */
      protected double[][] getCoefficients(){
  	double[][] coefficients = new double[m_numClasses][m_numericDataHeader.numAttributes() + 1];
  	for (int j = 0; j < m_numClasses; j++) {
  	    //go through simple regression functions and add their coefficient to the coefficient of
  	    //the attribute they are built on.
  	    for (int i = 0; i < m_numRegressions; i++) {
  		
  		double slope = m_regressions[j][i].getSlope();
  		double intercept = m_regressions[j][i].getIntercept();
  		int attribute = m_regressions[j][i].getAttributeIndex();
  		
  		coefficients[j][0] += intercept;
  		coefficients[j][attribute + 1] += slope;
  	    }
  	}
          
          // Need to multiply all coefficients by (J-1) / J
          for (int j = 0; j < coefficients.length; j++) {
            for (int i = 0; i < coefficients[0].length; i++) {
              coefficients[j][i] *= (double)(m_numClasses - 1) / (double)m_numClasses;
            }
          }
  
  	return coefficients;
      }
  
      /**
       * Returns the fraction of all attributes in the data that are used in the 
       * logistic model (in percent). 
       * An attribute is used in the model if it is used in any of the models for 
       * the different classes.
       * 
       * @return the fraction of all attributes that are used
       */
      public double percentAttributesUsed(){	
  	boolean[] attributes = new boolean[m_numericDataHeader.numAttributes()];
  	
  	double[][] coefficients = getCoefficients();
  	for (int j = 0; j < m_numClasses; j++){
  	    for (int i = 1; i < m_numericDataHeader.numAttributes() + 1; i++) {
  		//attribute used if it is used in any class, note coefficients are shifted by one (because
  		//of constant term).
  		if (!Utils.eq(coefficients[j][i],0)) attributes[i - 1] = true;
  	    }
  	}
  	
  	//count number of used attributes (without the class attribute)
  	double count = 0;
  	for (int i = 0; i < attributes.length; i++) if (attributes[i]) count++;
  	return count / (double)(m_numericDataHeader.numAttributes() - 1) * 100.0;
      }
      
      /**
       * Returns a description of the logistic model (i.e., attributes and 
       * coefficients).
       * 
       * @return the description of the model
       */
      public String toString(){
  	
  	StringBuffer s = new StringBuffer();	
  
  	//get used attributes
  	int[][] attributes = getUsedAttributes();
  	
  	//get coefficients
  	double[][] coefficients = getCoefficients();
  	
  	for (int j = 0; j < m_numClasses; j++) {
  	    s.append("\nClass "+j+" :\n");
  	    //constant term
  	    s.append(Utils.doubleToString(coefficients[j][0],4,2)+" + \n");
  	    for (int i = 0; i < attributes[j].length; i++) {		
  		//attribute/coefficient pairs
  		s.append("["+m_numericDataHeader.attribute(attributes[j][i]).name()+"]");
  		s.append(" * " + Utils.doubleToString(coefficients[j][attributes[j][i]+1],4,2));
  		if (i != attributes[j].length - 1) s.append(" +");
  		s.append("\n");	    
  	    }
  	}	
  	return new String(s);
      }
  
      /** 
       * Returns class probabilities for an instance.
       *
       * @param instance the instance to compute the distribution for
       * @return the class probabilities
       * @throws Exception if distribution can't be computed successfully
       */
      public double[] distributionForInstance(Instance instance) throws Exception {
  	
  	instance = (Instance)instance.copy();	
  
  	//set to numeric pseudo-class
        	instance.setDataset(m_numericDataHeader);		
  	
  	//calculate probs via Fs
  	return probs(getFs(instance));
      }
  
      /**
       * Cleanup in order to save memory.
       */
      public void cleanup() {
  	//save just header info
  	m_train = new Instances(m_train,0);
  	m_numericData = null;	
      }
      
      /**
       * Returns the revision string.
       * 
       * @return		the revision
       */
      public String getRevision() {
        return RevisionUtils.extract("$Revision: 5928 $");
      }
  }
  