content: /*
   *    This program is free software; you can redistribute it and/or modify
   *    it under the terms of the GNU General Public License as published by
   *    the Free Software Foundation; either version 2 of the License, or
   *    (at your option) any later version.
   *
   *    This program is distributed in the hope that it will be useful,
   *    but WITHOUT ANY WARRANTY; without even the implied warranty of
   *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   *    GNU General Public License for more details.
   *
   *    You should have received a copy of the GNU General Public License
   *    along with this program; if not, write to the Free Software
   *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
  
  /*
   *    CheckClassifier.java
   *    Copyright (C) 1999 University of Waikato, Hamilton, New Zealand
   *
   */
  
  package weka.classifiers;
  
  import weka.core.Attribute;
  import weka.core.CheckScheme;
  import weka.core.FastVector;
  import weka.core.Instance;
  import weka.core.Instances;
  import weka.core.MultiInstanceCapabilitiesHandler;
  import weka.core.Option;
  import weka.core.OptionHandler;
  import weka.core.RevisionUtils;
  import weka.core.SerializationHelper;
  import weka.core.TestInstances;
  import weka.core.Utils;
  import weka.core.WeightedInstancesHandler;
  
  import java.util.Enumeration;
  import java.util.Random;
  import java.util.Vector;
  
  /**
   * Class for examining the capabilities and finding problems with
   * classifiers. If you implement a classifier using the WEKA.libraries,
   * you should run the checks on it to ensure robustness and correct
   * operation. Passing all the tests of this object does not mean
   * bugs in the classifier don't exist, but this will help find some
   * common ones. <p/>
   *
   * Typical usage: <p/>
   * <code>java weka.classifiers.CheckClassifier -W classifier_name
   * classifier_options </code><p/>
   *
   * CheckClassifier reports on the following:
   * <ul>
   *    <li> Classifier abilities
   *      <ul>
   *         <li> Possible command line options to the classifier </li>
   *         <li> Whether the classifier can predict nominal, numeric, string,
   *              date or relational class attributes. Warnings will be displayed if
   *              performance is worse than ZeroR </li>
   *         <li> Whether the classifier can be trained incrementally </li>
   *         <li> Whether the classifier can handle numeric predictor attributes </li>
   *         <li> Whether the classifier can handle nominal predictor attributes </li>
   *         <li> Whether the classifier can handle string predictor attributes </li>
   *         <li> Whether the classifier can handle date predictor attributes </li>
   *         <li> Whether the classifier can handle relational predictor attributes </li>
   *         <li> Whether the classifier can handle multi-instance data </li>
   *         <li> Whether the classifier can handle missing predictor values </li>
   *         <li> Whether the classifier can handle missing class values </li>
   *         <li> Whether a nominal classifier only handles 2 class problems </li>
   *         <li> Whether the classifier can handle instance weights </li>
   *      </ul>
   *    </li>
   *    <li> Correct functioning
   *      <ul>
   *         <li> Correct initialisation during buildClassifier (i.e. no result
   *              changes when buildClassifier called repeatedly) </li>
   *         <li> Whether incremental training produces the same results
   *              as during non-incremental training (which may or may not
   *              be OK) </li>
   *         <li> Whether the classifier alters the data pased to it
   *              (number of instances, instance order, instance weights, etc) </li>
   *         <li> Whether the toString() method works correctly before the
   *              classifier has been built. </li>
   *      </ul>
   *    </li>
   *    <li> Degenerate cases
   *      <ul>
   *         <li> building classifier with zero training instances </li>
   *         <li> all but one predictor attribute values missing </li>
   *         <li> all predictor attribute values missing </li>
   *         <li> all but one class values missing </li>
   *         <li> all class values missing </li>
   *      </ul>
   *    </li>
   * </ul>
   * Running CheckClassifier with the debug option set will output the
   * training and test datasets for any failed tests.<p/>
   *
   * The <code>weka.classifiers.AbstractClassifierTest</code> uses this
   * class to test all the classifiers. Any changes here, have to be
   * checked in that abstract test class, too. <p/>
   *
   <!-- options-start -->
   * Valid options are: <p/>
   *
   * <pre> -D
   *  Turn on debugging output.</pre>
   *
   * <pre> -S
   *  Silent mode - prints nothing to stdout.</pre>
   *
   * <pre> -N &lt;num&gt;
   *  The number of instances in the datasets (default 20).</pre>
   *
   * <pre> -nominal &lt;num&gt;
   *  The number of nominal attributes (default 2).</pre>
   *
   * <pre> -nominal-values &lt;num&gt;
   *  The number of values for nominal attributes (default 1).</pre>
   *
   * <pre> -numeric &lt;num&gt;
   *  The number of numeric attributes (default 1).</pre>
   *
   * <pre> -string &lt;num&gt;
   *  The number of string attributes (default 1).</pre>
   *
   * <pre> -date &lt;num&gt;
   *  The number of date attributes (default 1).</pre>
   *
   * <pre> -relational &lt;num&gt;
   *  The number of relational attributes (default 1).</pre>
   *
   * <pre> -num-instances-relational &lt;num&gt;
   *  The number of instances in relational/bag attributes (default 10).</pre>
   *
   * <pre> -words &lt;comma-separated-list&gt;
   *  The words to use in string attributes.</pre>
   *
   * <pre> -word-separators &lt;chars&gt;
   *  The word separators to use in string attributes.</pre>
   *
   * <pre> -W
   *  Full name of the classifier analysed.
   *  eg: weka.classifiers.bayes.NaiveBayes
   *  (default weka.classifiers.rules.ZeroR)</pre>
   *
   * <pre>
   * Options specific to classifier weka.classifiers.rules.ZeroR:
   * </pre>
   *
   * <pre> -D
   *  If set, classifier is run in debug mode and
   *  may output additional info to the console</pre>
   *
   <!-- options-end -->
   *
   * Options after -- are passed to the designated classifier.<p/>
   *
   * @author Len Trigg (trigg@cs.waikato.ac.nz)
   * @author FracPete (fracpete at waikato dot ac dot nz)
   * @version $Revision: 6041 $
   * @see TestInstances
   */
  public class CheckClassifier
    extends CheckScheme {
  
    /*
     * Note about test methods:
     * - methods return array of booleans
     * - first index: success or not
     * - second index: acceptable or not (e.g., Exception is OK)
     * - in case the performance is worse than that of ZeroR both indices are true
     *
     * FracPete (fracpete at waikato dot ac dot nz)
     */
  
    /*** The classifier to be examined */
    protected Classifier m_Classifier = new weka.classifiers.rules.ZeroR();
  
    /**
     * Returns an enumeration describing the available options.
     *
     * @return an enumeration of all the available options.
     */
    public Enumeration listOptions() {
      Vector result = new Vector();
  
      Enumeration en = super.listOptions();
      while (en.hasMoreElements())
        result.addElement(en.nextElement());
  
      result.addElement(new Option(
          "\tFull name of the classifier analysed.\n"
          +"\teg: weka.classifiers.bayes.NaiveBayes\n"
          + "\t(default weka.classifiers.rules.ZeroR)",
          "W", 1, "-W"));
  
      if ((m_Classifier != null)
          && (m_Classifier instanceof OptionHandler)) {
        result.addElement(new Option("", "", 0,
            "\nOptions specific to classifier "
            + m_Classifier.getClass().getName()
            + ":"));
        Enumeration enu = ((OptionHandler)m_Classifier).listOptions();
        while (enu.hasMoreElements())
          result.addElement(enu.nextElement());
      }
  
      return result.elements();
    }
  
    /**
     * Parses a given list of options.
     *
     <!-- options-start -->
     * Valid options are: <p/>
     *
     * <pre> -D
     *  Turn on debugging output.</pre>
     *
     * <pre> -S
     *  Silent mode - prints nothing to stdout.</pre>
     *
     * <pre> -N &lt;num&gt;
     *  The number of instances in the datasets (default 20).</pre>
     *
     * <pre> -nominal &lt;num&gt;
     *  The number of nominal attributes (default 2).</pre>
     *
     * <pre> -nominal-values &lt;num&gt;
     *  The number of values for nominal attributes (default 1).</pre>
     *
     * <pre> -numeric &lt;num&gt;
     *  The number of numeric attributes (default 1).</pre>
     *
     * <pre> -string &lt;num&gt;
     *  The number of string attributes (default 1).</pre>
     *
     * <pre> -date &lt;num&gt;
     *  The number of date attributes (default 1).</pre>
     *
     * <pre> -relational &lt;num&gt;
     *  The number of relational attributes (default 1).</pre>
     *
     * <pre> -num-instances-relational &lt;num&gt;
     *  The number of instances in relational/bag attributes (default 10).</pre>
     *
     * <pre> -words &lt;comma-separated-list&gt;
     *  The words to use in string attributes.</pre>
     *
     * <pre> -word-separators &lt;chars&gt;
     *  The word separators to use in string attributes.</pre>
     *
     * <pre> -W
     *  Full name of the classifier analysed.
     *  eg: weka.classifiers.bayes.NaiveBayes
     *  (default weka.classifiers.rules.ZeroR)</pre>
     *
     * <pre>
     * Options specific to classifier weka.classifiers.rules.ZeroR:
     * </pre>
     *
     * <pre> -D
     *  If set, classifier is run in debug mode and
     *  may output additional info to the console</pre>
     *
     <!-- options-end -->
     *
     * @param options the list of options as an array of strings
     * @throws Exception if an option is not supported
     */
    public void setOptions(String[] options) throws Exception {
      String      tmpStr;
  
      super.setOptions(options);
  
      tmpStr = Utils.getOption('W', options);
      if (tmpStr.length() == 0)
        tmpStr = weka.classifiers.rules.ZeroR.class.getName();
      setClassifier(
          (Classifier) forName(
            "weka.classifiers",
            Classifier.class,
            tmpStr,
            Utils.partitionOptions(options)));
    }
  
    /**
     * Gets the current settings of the CheckClassifier.
     *
     * @return an array of strings suitable for passing to setOptions
     */
    public String[] getOptions() {
      Vector        result;
      String[]      options;
      int           i;
  
      result = new Vector();
  
      options = super.getOptions();
      for (i = 0; i < options.length; i++)
        result.add(options[i]);
  
      if (getClassifier() != null) {
        result.add("-W");
        result.add(getClassifier().getClass().getName());
      }
  
      if ((m_Classifier != null) && (m_Classifier instanceof OptionHandler))
        options = ((OptionHandler) m_Classifier).getOptions();
      else
        options = new String[0];
  
      if (options.length > 0) {
        result.add("--");
        for (i = 0; i < options.length; i++)
          result.add(options[i]);
      }
  
      return (String[]) result.toArray(new String[result.size()]);
    }
  
    /**
     * Begin the tests, reporting results to System.out
     */
    public void doTests() {
  
      if (getClassifier() == null) {
        println("\n=== No classifier set ===");
        return;
      }
      println("\n=== Check on Classifier: "
          + getClassifier().getClass().getName()
          + " ===\n");
  
      // Start tests
      m_ClasspathProblems = false;
      println("--> Checking for interfaces");
      canTakeOptions();
      boolean updateableClassifier = updateableClassifier()[0];
      boolean weightedInstancesHandler = weightedInstancesHandler()[0];
      boolean multiInstanceHandler = multiInstanceHandler()[0];
      println("--> Classifier tests");
      declaresSerialVersionUID();
      testToString();
      testsPerClassType(Attribute.NOMINAL,    updateableClassifier, weightedInstancesHandler, multiInstanceHandler);
      testsPerClassType(Attribute.NUMERIC,    updateableClassifier, weightedInstancesHandler, multiInstanceHandler);
      testsPerClassType(Attribute.DATE,       updateableClassifier, weightedInstancesHandler, multiInstanceHandler);
      testsPerClassType(Attribute.STRING,     updateableClassifier, weightedInstancesHandler, multiInstanceHandler);
      testsPerClassType(Attribute.RELATIONAL, updateableClassifier, weightedInstancesHandler, multiInstanceHandler);
    }
  
    /**
     * Set the classifier for boosting.
     *
     * @param newClassifier the Classifier to use.
     */
    public void setClassifier(Classifier newClassifier) {
      m_Classifier = newClassifier;
    }
  
    /**
     * Get the classifier used as the classifier
     *
     * @return the classifier used as the classifier
     */
    public Classifier getClassifier() {
      return m_Classifier;
    }
  
    /**
     * Run a battery of tests for a given class attribute type
     *
     * @param classType true if the class attribute should be numeric
     * @param updateable true if the classifier is updateable
     * @param weighted true if the classifier says it handles weights
     * @param multiInstance true if the classifier is a multi-instance classifier
     */
    protected void testsPerClassType(int classType,
                                     boolean updateable,
                                     boolean weighted,
                                     boolean multiInstance) {
  
      boolean PNom = canPredict(true,  false, false, false, false, multiInstance, classType)[0];
      boolean PNum = canPredict(false, true,  false, false, false, multiInstance, classType)[0];
      boolean PStr = canPredict(false, false, true,  false, false, multiInstance, classType)[0];
      boolean PDat = canPredict(false, false, false, true,  false, multiInstance, classType)[0];
      boolean PRel;
      if (!multiInstance)
        PRel = canPredict(false, false, false, false,  true, multiInstance, classType)[0];
      else
        PRel = false;
  
      if (PNom || PNum || PStr || PDat || PRel) {
        if (weighted)
          instanceWeights(PNom, PNum, PStr, PDat, PRel, multiInstance, classType);
  
        canHandleOnlyClass(PNom, PNum, PStr, PDat, PRel, classType);
  
        if (classType == Attribute.NOMINAL)
          canHandleNClasses(PNom, PNum, PStr, PDat, PRel, multiInstance, 4);
  
        if (!multiInstance) {
          canHandleClassAsNthAttribute(PNom, PNum, PStr, PDat, PRel, multiInstance, classType, 0);
          canHandleClassAsNthAttribute(PNom, PNum, PStr, PDat, PRel, multiInstance, classType, 1);
        }
  
        canHandleZeroTraining(PNom, PNum, PStr, PDat, PRel, multiInstance, classType);
        boolean handleMissingPredictors = canHandleMissing(PNom, PNum, PStr, PDat, PRel,
            multiInstance, classType,
            true, false, 20)[0];
        if (handleMissingPredictors)
          canHandleMissing(PNom, PNum, PStr, PDat, PRel, multiInstance, classType, true, false, 100);
  
        boolean handleMissingClass = canHandleMissing(PNom, PNum, PStr, PDat, PRel,
            multiInstance, classType,
            false, true, 20)[0];
        if (handleMissingClass)
          canHandleMissing(PNom, PNum, PStr, PDat, PRel, multiInstance, classType, false, true, 100);
  
        correctBuildInitialisation(PNom, PNum, PStr, PDat, PRel, multiInstance, classType);
        datasetIntegrity(PNom, PNum, PStr, PDat, PRel, multiInstance, classType,
            handleMissingPredictors, handleMissingClass);
        doesntUseTestClassVal(PNom, PNum, PStr, PDat, PRel, multiInstance, classType);
        if (updateable)
          updatingEquality(PNom, PNum, PStr, PDat, PRel, multiInstance, classType);
      }
    }
  
    /**
     * Checks whether the scheme's toString() method works even though the
     * classifies hasn't been built yet.
     *
     * @return index 0 is true if the toString() method works fine
     */
    protected boolean[] testToString() {
      boolean[] result = new boolean[2];
  
      print("toString...");
  
      try {
        Classifier copy = (Classifier) m_Classifier.getClass().newInstance();
        copy.toString();
        result[0] = true;
        println("yes");
      }
      catch (Exception e) {
        result[0] = false;
        println("no");
        if (m_Debug) {
          println("\n=== Full report ===");
          e.printStackTrace();
          println("\n");
        }
      }
  
      return result;
    }
  
    /**
     * tests for a serialVersionUID. Fails in case the scheme doesn't declare
     * a UID.
     *
     * @return index 0 is true if the scheme declares a UID
     */
    protected boolean[] declaresSerialVersionUID() {
      boolean[] result = new boolean[2];
  
      print("serialVersionUID...");
  
      result[0] = !SerializationHelper.needsUID(m_Classifier.getClass());
  
      if (result[0])
        println("yes");
      else
        println("no");
  
      return result;
    }
  
    /**
     * Checks whether the scheme can take command line options.
     *
     * @return index 0 is true if the classifier can take options
     */
    protected boolean[] canTakeOptions() {
  
      boolean[] result = new boolean[2];
  
      print("options...");
      if (m_Classifier instanceof OptionHandler) {
        println("yes");
        if (m_Debug) {
          println("\n=== Full report ===");
          Enumeration enu = ((OptionHandler)m_Classifier).listOptions();
          while (enu.hasMoreElements()) {
            Option option = (Option) enu.nextElement();
            print(option.synopsis() + "\n"
                + option.description() + "\n");
          }
          println("\n");
        }
        result[0] = true;
      }
      else {
        println("no");
        result[0] = false;
      }
  
      return result;
    }
  
    /**
     * Checks whether the scheme can build models incrementally.
     *
     * @return index 0 is true if the classifier can train incrementally
     */
    protected boolean[] updateableClassifier() {
  
      boolean[] result = new boolean[2];
  
      print("updateable classifier...");
      if (m_Classifier instanceof UpdateableClassifier) {
        println("yes");
        result[0] = true;
      }
      else {
        println("no");
        result[0] = false;
      }
  
      return result;
    }
  
    /**
     * Checks whether the scheme says it can handle instance weights.
     *
     * @return true if the classifier handles instance weights
     */
    protected boolean[] weightedInstancesHandler() {
  
      boolean[] result = new boolean[2];
  
      print("weighted instances classifier...");
      if (m_Classifier instanceof WeightedInstancesHandler) {
        println("yes");
        result[0] = true;
      }
      else {
        println("no");
        result[0] = false;
      }
  
      return result;
    }
  
    /**
     * Checks whether the scheme handles multi-instance data.
     *
     * @return true if the classifier handles multi-instance data
     */
    protected boolean[] multiInstanceHandler() {
      boolean[] result = new boolean[2];
  
      print("multi-instance classifier...");
      if (m_Classifier instanceof MultiInstanceCapabilitiesHandler) {
        println("yes");
        result[0] = true;
      }
      else {
        println("no");
        result[0] = false;
      }
  
      return result;
    }
  
    /**
     * Checks basic prediction of the scheme, for simple non-troublesome
     * datasets.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NOMINAL, NUMERIC, etc.)
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] canPredict(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      print("basic predict");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      FastVector accepts = new FastVector();
      accepts.addElement("unary");
      accepts.addElement("binary");
      accepts.addElement("nominal");
      accepts.addElement("numeric");
      accepts.addElement("string");
      accepts.addElement("date");
      accepts.addElement("relational");
      accepts.addElement("multi-instance");
      accepts.addElement("not in classpath");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      return runBasicTest(nominalPredictor, numericPredictor, stringPredictor,
          datePredictor, relationalPredictor,
          multiInstance,
          classType,
          missingLevel, predictorMissing, classMissing,
          numTrain, numTest, numClasses,
          accepts);
    }
  
    /**
     * Checks whether the scheme can handle data that contains only the class
     * attribute. If a scheme cannot build a proper model with that data, it
     * should default back to a ZeroR model.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param classType the class type (NOMINAL, NUMERIC, etc.)
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] canHandleOnlyClass(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        int classType) {
  
      print("only class in data");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, false, classType);
      print("...");
      FastVector accepts = new FastVector();
      accepts.addElement("class");
      accepts.addElement("zeror");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      return runBasicTest(false, false, false, false, false,
                          false,
                          classType,
                          missingLevel, predictorMissing, classMissing,
                          numTrain, numTest, 2,
                          accepts);
    }
  
    /**
     * Checks whether nominal schemes can handle more than two classes.
     * If a scheme is only designed for two-class problems it should
     * throw an appropriate exception for multi-class problems.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param numClasses the number of classes to test
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] canHandleNClasses(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int numClasses) {
  
      print("more than two class problems");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, Attribute.NOMINAL);
      print("...");
      FastVector accepts = new FastVector();
      accepts.addElement("number");
      accepts.addElement("class");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      return runBasicTest(nominalPredictor, numericPredictor, stringPredictor,
                          datePredictor, relationalPredictor,
                          multiInstance,
                          Attribute.NOMINAL,
                          missingLevel, predictorMissing, classMissing,
                          numTrain, numTest, numClasses,
                          accepts);
    }
  
    /**
     * Checks whether the scheme can handle class attributes as Nth attribute.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param classIndex the index of the class attribute (0-based, -1 means last attribute)
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     * @see TestInstances#CLASS_IS_LAST
     */
    protected boolean[] canHandleClassAsNthAttribute(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType,
        int classIndex) {
  
      if (classIndex == TestInstances.CLASS_IS_LAST)
        print("class attribute as last attribute");
      else
        print("class attribute as " + (classIndex + 1) + ". attribute");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      FastVector accepts = new FastVector();
      int numTrain = getNumInstances(), numTest = getNumInstances(), numClasses = 2,
      missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      return runBasicTest(nominalPredictor, numericPredictor, stringPredictor,
                          datePredictor, relationalPredictor,
                          multiInstance,
                          classType,
                          classIndex,
                          missingLevel, predictorMissing, classMissing,
                          numTrain, numTest, numClasses,
                          accepts);
    }
  
    /**
     * Checks whether the scheme can handle zero training instances.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] canHandleZeroTraining(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      print("handle zero training instances");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      FastVector accepts = new FastVector();
      accepts.addElement("train");
      accepts.addElement("value");
      int numTrain = 0, numTest = getNumInstances(), numClasses = 2,
      missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      return runBasicTest(
                nominalPredictor, numericPredictor, stringPredictor,
                datePredictor, relationalPredictor,
                multiInstance,
                classType,
                missingLevel, predictorMissing, classMissing,
                numTrain, numTest, numClasses,
                accepts);
    }
  
    /**
     * Checks whether the scheme correctly initialises models when
     * buildClassifier is called. This test calls buildClassifier with
     * one training dataset and records performance on a test set.
     * buildClassifier is then called on a training set with different
     * structure, and then again with the original training set. The
     * performance on the test set is compared with the original results
     * and any performance difference noted as incorrect build initialisation.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @return index 0 is true if the test was passed, index 1 is true if the
     *         scheme performs worse than ZeroR, but without error (index 0 is
     *         false)
     */
    protected boolean[] correctBuildInitialisation(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      boolean[] result = new boolean[2];
  
      print("correct initialisation during buildClassifier");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      Instances train1 = null;
      Instances test1 = null;
      Instances train2 = null;
      Instances test2 = null;
      Classifier classifier = null;
      Evaluation evaluation1A = null;
      Evaluation evaluation1B = null;
      Evaluation evaluation2 = null;
      boolean built = false;
      int stage = 0;
      try {
  
        // Make two sets of train/test splits with different
        // numbers of attributes
        train1 = makeTestDataset(42, numTrain,
                                 nominalPredictor    ? getNumNominal()    : 0,
                                 numericPredictor    ? getNumNumeric()    : 0,
                                 stringPredictor     ? getNumString()     : 0,
                                 datePredictor       ? getNumDate()       : 0,
                                 relationalPredictor ? getNumRelational() : 0,
                                 numClasses,
                                 classType,
                                 multiInstance);
        train2 = makeTestDataset(84, numTrain,
                                 nominalPredictor    ? getNumNominal() + 1 : 0,
                                 numericPredictor    ? getNumNumeric() + 1 : 0,
                                 stringPredictor     ? getNumString()      : 0,
                                 datePredictor       ? getNumDate()        : 0,
                                 relationalPredictor ? getNumRelational()  : 0,
                                 numClasses,
                                 classType,
                                 multiInstance);
        test1 = makeTestDataset(24, numTest,
                                nominalPredictor     ? getNumNominal()    : 0,
                                numericPredictor     ? getNumNumeric()    : 0,
                                stringPredictor      ? getNumString()     : 0,
                                datePredictor        ? getNumDate()       : 0,
                                relationalPredictor  ? getNumRelational() : 0,
                                numClasses,
                                classType,
                                multiInstance);
        test2 = makeTestDataset(48, numTest,
                                nominalPredictor    ? getNumNominal() + 1 : 0,
                                numericPredictor    ? getNumNumeric() + 1 : 0,
                                stringPredictor     ? getNumString()      : 0,
                                datePredictor       ? getNumDate()        : 0,
                                relationalPredictor ? getNumRelational()  : 0,
                                numClasses,
                                classType,
                                multiInstance);
        if (missingLevel > 0) {
          addMissing(train1, missingLevel, predictorMissing, classMissing);
          addMissing(test1, Math.min(missingLevel,50), predictorMissing,
              classMissing);
          addMissing(train2, missingLevel, predictorMissing, classMissing);
          addMissing(test2, Math.min(missingLevel,50), predictorMissing,
              classMissing);
        }
  
        classifier = AbstractClassifier.makeCopies(getClassifier(), 1)[0];
        evaluation1A = new Evaluation(train1);
        evaluation1B = new Evaluation(train1);
        evaluation2 = new Evaluation(train2);
      } catch (Exception ex) {
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
        stage = 0;
        classifier.buildClassifier(train1);
        built = true;
        if (!testWRTZeroR(classifier, evaluation1A, train1, test1)[0]) {
          throw new Exception("Scheme performs worse than ZeroR");
        }
  
        stage = 1;
        built = false;
        classifier.buildClassifier(train2);
        built = true;
        if (!testWRTZeroR(classifier, evaluation2, train2, test2)[0]) {
          throw new Exception("Scheme performs worse than ZeroR");
        }
  
        stage = 2;
        built = false;
        classifier.buildClassifier(train1);
        built = true;
        if (!testWRTZeroR(classifier, evaluation1B, train1, test1)[0]) {
          throw new Exception("Scheme performs worse than ZeroR");
        }
  
        stage = 3;
        if (!evaluation1A.equals(evaluation1B)) {
          if (m_Debug) {
            println("\n=== Full report ===\n"
                + evaluation1A.toSummaryString("\nFirst buildClassifier()",
                    true)
                    + "\n\n");
            println(
                evaluation1B.toSummaryString("\nSecond buildClassifier()",
                    true)
                    + "\n\n");
          }
          throw new Exception("Results differ between buildClassifier calls");
        }
        println("yes");
        result[0] = true;
  
        if (false && m_Debug) {
          println("\n=== Full report ===\n"
              + evaluation1A.toSummaryString("\nFirst buildClassifier()",
                  true)
                  + "\n\n");
          println(
              evaluation1B.toSummaryString("\nSecond buildClassifier()",
                  true)
                  + "\n\n");
        }
      }
      catch (Exception ex) {
        String msg = ex.getMessage().toLowerCase();
        if (msg.indexOf("worse than zeror") >= 0) {
          println("warning: performs worse than ZeroR");
          result[0] = (stage < 1);
          result[1] = (stage < 1);
        } else {
          println("no");
          result[0] = false;
        }
        if (m_Debug) {
          println("\n=== Full Report ===");
          print("Problem during");
          if (built) {
            print(" testing");
          } else {
            print(" training");
          }
          switch (stage) {
            case 0:
              print(" of dataset 1");
              break;
            case 1:
              print(" of dataset 2");
              break;
            case 2:
              print(" of dataset 1 (2nd build)");
              break;
            case 3:
              print(", comparing results from builds of dataset 1");
              break;
          }
          println(": " + ex.getMessage() + "\n");
          println("here are the datasets:\n");
          println("=== Train1 Dataset ===\n"
              + train1.toString() + "\n");
          println("=== Test1 Dataset ===\n"
              + test1.toString() + "\n\n");
          println("=== Train2 Dataset ===\n"
              + train2.toString() + "\n");
          println("=== Test2 Dataset ===\n"
              + test2.toString() + "\n\n");
        }
      }
  
      return result;
    }
  
    /**
     * Checks basic missing value handling of the scheme. If the missing
     * values cause an exception to be thrown by the scheme, this will be
     * recorded.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param predictorMissing true if the missing values may be in
     * the predictors
     * @param classMissing true if the missing values may be in the class
     * @param missingLevel the percentage of missing values
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] canHandleMissing(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType,
        boolean predictorMissing,
        boolean classMissing,
        int missingLevel) {
  
      if (missingLevel == 100)
        print("100% ");
      print("missing");
      if (predictorMissing) {
        print(" predictor");
        if (classMissing)
          print(" and");
      }
      if (classMissing)
        print(" class");
      print(" values");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      FastVector accepts = new FastVector();
      accepts.addElement("missing");
      accepts.addElement("value");
      accepts.addElement("train");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      numClasses = 2;
  
      return runBasicTest(nominalPredictor, numericPredictor, stringPredictor,
          datePredictor, relationalPredictor,
          multiInstance,
          classType,
          missingLevel, predictorMissing, classMissing,
          numTrain, numTest, numClasses,
          accepts);
    }
  
    /**
     * Checks whether an updateable scheme produces the same model when
     * trained incrementally as when batch trained. The model itself
     * cannot be compared, so we compare the evaluation on test data
     * for both models. It is possible to get a false positive on this
     * test (likelihood depends on the classifier).
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @return index 0 is true if the test was passed
     */
    protected boolean[] updatingEquality(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      print("incremental training produces the same results"
          + " as batch training");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      boolean[] result = new boolean[2];
      Instances train = null;
      Instances test = null;
      Classifier [] classifiers = null;
      Evaluation evaluationB = null;
      Evaluation evaluationI = null;
      boolean built = false;
      try {
        train = makeTestDataset(42, numTrain,
                                nominalPredictor    ? getNumNominal()    : 0,
                                numericPredictor    ? getNumNumeric()    : 0,
                                stringPredictor     ? getNumString()     : 0,
                                datePredictor       ? getNumDate()       : 0,
                                relationalPredictor ? getNumRelational() : 0,
                                numClasses,
                                classType,
                                multiInstance);
        test = makeTestDataset(24, numTest,
                               nominalPredictor    ? getNumNominal()    : 0,
                               numericPredictor    ? getNumNumeric()    : 0,
                               stringPredictor     ? getNumString()     : 0,
                               datePredictor       ? getNumDate()       : 0,
                               relationalPredictor ? getNumRelational() : 0,
                               numClasses,
                               classType,
                               multiInstance);
        if (missingLevel > 0) {
          addMissing(train, missingLevel, predictorMissing, classMissing);
          addMissing(test, Math.min(missingLevel, 50), predictorMissing,
              classMissing);
        }
        classifiers = AbstractClassifier.makeCopies(getClassifier(), 2);
        evaluationB = new Evaluation(train);
        evaluationI = new Evaluation(train);
        classifiers[0].buildClassifier(train);
        testWRTZeroR(classifiers[0], evaluationB, train, test);
      } catch (Exception ex) {
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
        classifiers[1].buildClassifier(new Instances(train, 0));
        for (int i = 0; i < train.numInstances(); i++) {
          ((UpdateableClassifier)classifiers[1]).updateClassifier(
              train.instance(i));
        }
        built = true;
        testWRTZeroR(classifiers[1], evaluationI, train, test);
        if (!evaluationB.equals(evaluationI)) {
          println("no");
          result[0] = false;
  
          if (m_Debug) {
            println("\n=== Full Report ===");
            println("Results differ between batch and "
                + "incrementally built models.\n"
                + "Depending on the classifier, this may be OK");
            println("Here are the results:\n");
            println(evaluationB.toSummaryString(
                "\nbatch built results\n", true));
            println(evaluationI.toSummaryString(
                "\nincrementally built results\n", true));
            println("Here are the datasets:\n");
            println("=== Train Dataset ===\n"
                + train.toString() + "\n");
            println("=== Test Dataset ===\n"
                + test.toString() + "\n\n");
          }
        }
        else {
          println("yes");
          result[0] = true;
        }
      } catch (Exception ex) {
        result[0] = false;
  
        print("Problem during");
        if (built)
          print(" testing");
        else
          print(" training");
        println(": " + ex.getMessage() + "\n");
      }
  
      return result;
    }
  
    /**
     * Checks whether the classifier erroneously uses the class
     * value of test instances (if provided). Runs the classifier with
     * test instance class values set to missing and compares with results
     * when test instance class values are left intact.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @return index 0 is true if the test was passed
     */
    protected boolean[] doesntUseTestClassVal(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      print("classifier ignores test instance class vals");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      int numTrain = 2*getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      boolean[] result = new boolean[2];
      Instances train = null;
      Instances test = null;
      Classifier [] classifiers = null;
      boolean evalFail = false;
      try {
        train = makeTestDataset(42, numTrain,
                                nominalPredictor    ? getNumNominal() + 1 : 0,
                                numericPredictor    ? getNumNumeric() + 1 : 0,
                                stringPredictor     ? getNumString()      : 0,
                                datePredictor       ? getNumDate()        : 0,
                                relationalPredictor ? getNumRelational()  : 0,
                                numClasses,
                                classType,
                                multiInstance);
        test = makeTestDataset(24, numTest,
                               nominalPredictor    ? getNumNominal() + 1 : 0,
                               numericPredictor    ? getNumNumeric() + 1 : 0,
                               stringPredictor     ? getNumString()      : 0,
                               datePredictor       ? getNumDate()        : 0,
                               relationalPredictor ? getNumRelational()  : 0,
                               numClasses,
                               classType,
                               multiInstance);
        if (missingLevel > 0) {
          addMissing(train, missingLevel, predictorMissing, classMissing);
          addMissing(test, Math.min(missingLevel, 50), predictorMissing,
              classMissing);
        }
        classifiers = AbstractClassifier.makeCopies(getClassifier(), 2);
        classifiers[0].buildClassifier(train);
        classifiers[1].buildClassifier(train);
      } catch (Exception ex) {
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
  
        // Now set test values to missing when predicting
        for (int i = 0; i < test.numInstances(); i++) {
          Instance testInst = test.instance(i);
          Instance classMissingInst = (Instance)testInst.copy();
          classMissingInst.setDataset(test);
          classMissingInst.setClassMissing();
          double [] dist0 = classifiers[0].distributionForInstance(testInst);
          double [] dist1 = classifiers[1].distributionForInstance(classMissingInst);
          for (int j = 0; j < dist0.length; j++) {
            // ignore, if both are NaNs
            if (Double.isNaN(dist0[j]) && Double.isNaN(dist1[j])) {
              if (getDebug())
                System.out.println("Both predictions are NaN!");
              continue;
            }
            // distribution different?
            if (dist0[j] != dist1[j]) {
              throw new Exception("Prediction different for instance " + (i + 1));
            }
          }
        }
  
        println("yes");
        result[0] = true;
      } catch (Exception ex) {
        println("no");
        result[0] = false;
  
        if (m_Debug) {
          println("\n=== Full Report ===");
  
          if (evalFail) {
            println("Results differ between non-missing and "
                + "missing test class values.");
          } else {
            print("Problem during testing");
            println(": " + ex.getMessage() + "\n");
          }
          println("Here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Train Weights ===\n");
          for (int i = 0; i < train.numInstances(); i++) {
            println(" " + (i + 1)
                + "    " + train.instance(i).weight());
          }
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
          println("(test weights all 1.0\n");
        }
      }
  
      return result;
    }
  
    /**
     * Checks whether the classifier can handle instance weights.
     * This test compares the classifier performance on two datasets
     * that are identical except for the training weights. If the
     * results change, then the classifier must be using the weights. It
     * may be possible to get a false positive from this test if the
     * weight changes aren't significant enough to induce a change
     * in classifier performance (but the weights are chosen to minimize
     * the likelihood of this).
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @return index 0 true if the test was passed
     */
    protected boolean[] instanceWeights(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType) {
  
      print("classifier uses instance weights");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      int numTrain = 2*getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 0;
      boolean predictorMissing = false, classMissing = false;
  
      boolean[] result = new boolean[2];
      Instances train = null;
      Instances test = null;
      Classifier [] classifiers = null;
      Evaluation evaluationB = null;
      Evaluation evaluationI = null;
      boolean built = false;
      boolean evalFail = false;
      try {
        train = makeTestDataset(42, numTrain,
                                nominalPredictor    ? getNumNominal() + 1 : 0,
                                numericPredictor    ? getNumNumeric() + 1 : 0,
                                stringPredictor     ? getNumString()      : 0,
                                datePredictor       ? getNumDate()        : 0,
                                relationalPredictor ? getNumRelational()  : 0,
                                numClasses,
                                classType,
                                multiInstance);
        test = makeTestDataset(24, numTest,
                               nominalPredictor    ? getNumNominal() + 1 : 0,
                               numericPredictor    ? getNumNumeric() + 1 : 0,
                               stringPredictor     ? getNumString()      : 0,
                               datePredictor       ? getNumDate()        : 0,
                               relationalPredictor ? getNumRelational()  : 0,
                               numClasses,
                               classType,
                               multiInstance);
        if (missingLevel > 0) {
          addMissing(train, missingLevel, predictorMissing, classMissing);
          addMissing(test, Math.min(missingLevel, 50), predictorMissing,
              classMissing);
        }
        classifiers = AbstractClassifier.makeCopies(getClassifier(), 2);
        evaluationB = new Evaluation(train);
        evaluationI = new Evaluation(train);
        classifiers[0].buildClassifier(train);
        testWRTZeroR(classifiers[0], evaluationB, train, test);
      } catch (Exception ex) {
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
  
        // Now modify instance weights and re-built/test
        for (int i = 0; i < train.numInstances(); i++) {
          train.instance(i).setWeight(0);
        }
        Random random = new Random(1);
        for (int i = 0; i < train.numInstances() / 2; i++) {
          int inst = Math.abs(random.nextInt()) % train.numInstances();
          int weight = Math.abs(random.nextInt()) % 10 + 1;
          train.instance(inst).setWeight(weight);
        }
        classifiers[1].buildClassifier(train);
        built = true;
        testWRTZeroR(classifiers[1], evaluationI, train, test);
        if (evaluationB.equals(evaluationI)) {
          // println("no");
          evalFail = true;
          throw new Exception("evalFail");
        }
  
        println("yes");
        result[0] = true;
      } catch (Exception ex) {
        println("no");
        result[0] = false;
  
        if (m_Debug) {
          println("\n=== Full Report ===");
  
          if (evalFail) {
            println("Results don't differ between non-weighted and "
                + "weighted instance models.");
            println("Here are the results:\n");
            println(evaluationB.toSummaryString("\nboth methods\n",
                true));
          } else {
            print("Problem during");
            if (built) {
              print(" testing");
            } else {
              print(" training");
            }
            println(": " + ex.getMessage() + "\n");
          }
          println("Here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Train Weights ===\n");
          for (int i = 0; i < train.numInstances(); i++) {
            println(" " + (i + 1)
                + "    " + train.instance(i).weight());
          }
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
          println("(test weights all 1.0\n");
        }
      }
  
      return result;
    }
  
    /**
     * Checks whether the scheme alters the training dataset during
     * training. If the scheme needs to modify the training
     * data it should take a copy of the training data. Currently checks
     * for changes to header structure, number of instances, order of
     * instances, instance weights.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param predictorMissing true if we know the classifier can handle
     * (at least) moderate missing predictor values
     * @param classMissing true if we know the classifier can handle
     * (at least) moderate missing class values
     * @return index 0 is true if the test was passed
     */
    protected boolean[] datasetIntegrity(
        boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType,
        boolean predictorMissing,
        boolean classMissing) {
  
      print("classifier doesn't alter original datasets");
      printAttributeSummary(
          nominalPredictor, numericPredictor, stringPredictor, datePredictor, relationalPredictor, multiInstance, classType);
      print("...");
      int numTrain = getNumInstances(), numTest = getNumInstances(),
      numClasses = 2, missingLevel = 20;
  
      boolean[] result = new boolean[2];
      Instances train = null;
      Instances test = null;
      Classifier classifier = null;
      Evaluation evaluation = null;
      boolean built = false;
      try {
        train = makeTestDataset(42, numTrain,
                                nominalPredictor    ? getNumNominal()    : 0,
                                numericPredictor    ? getNumNumeric()    : 0,
                                stringPredictor     ? getNumString()     : 0,
                                datePredictor       ? getNumDate()       : 0,
                                relationalPredictor ? getNumRelational() : 0,
                                numClasses,
                                classType,
                                multiInstance);
        test = makeTestDataset(24, numTest,
                               nominalPredictor     ? getNumNominal()    : 0,
                               numericPredictor     ? getNumNumeric()    : 0,
                               stringPredictor      ? getNumString()     : 0,
                               datePredictor        ? getNumDate()       : 0,
                               relationalPredictor  ? getNumRelational() : 0,
                               numClasses,
                               classType,
                               multiInstance);
        if (missingLevel > 0) {
          addMissing(train, missingLevel, predictorMissing, classMissing);
          addMissing(test, Math.min(missingLevel, 50), predictorMissing,
              classMissing);
        }
        classifier = AbstractClassifier.makeCopies(getClassifier(), 1)[0];
        evaluation = new Evaluation(train);
      } catch (Exception ex) {
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
        Instances trainCopy = new Instances(train);
        Instances testCopy = new Instances(test);
        classifier.buildClassifier(trainCopy);
        compareDatasets(train, trainCopy);
        built = true;
        testWRTZeroR(classifier, evaluation, trainCopy, testCopy);
        compareDatasets(test, testCopy);
  
        println("yes");
        result[0] = true;
      } catch (Exception ex) {
        println("no");
        result[0] = false;
  
        if (m_Debug) {
          println("\n=== Full Report ===");
          print("Problem during");
          if (built) {
            print(" testing");
          } else {
            print(" training");
          }
          println(": " + ex.getMessage() + "\n");
          println("Here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
        }
      }
  
      return result;
    }
  
    /**
     * Runs a text on the datasets with the given characteristics.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param missingLevel the percentage of missing values
     * @param predictorMissing true if the missing values may be in
     * the predictors
     * @param classMissing true if the missing values may be in the class
     * @param numTrain the number of instances in the training set
     * @param numTest the number of instaces in the test set
     * @param numClasses the number of classes
     * @param accepts the acceptable string in an exception
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] runBasicTest(boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType,
        int missingLevel,
        boolean predictorMissing,
        boolean classMissing,
        int numTrain,
        int numTest,
        int numClasses,
        FastVector accepts) {
  
      return runBasicTest(
          nominalPredictor,
          numericPredictor,
          stringPredictor,
          datePredictor,
          relationalPredictor,
          multiInstance,
          classType,
          TestInstances.CLASS_IS_LAST,
          missingLevel,
          predictorMissing,
          classMissing,
          numTrain,
          numTest,
          numClasses,
          accepts);
    }
  
    /**
     * Runs a text on the datasets with the given characteristics.
     *
     * @param nominalPredictor if true use nominal predictor attributes
     * @param numericPredictor if true use numeric predictor attributes
     * @param stringPredictor if true use string predictor attributes
     * @param datePredictor if true use date predictor attributes
     * @param relationalPredictor if true use relational predictor attributes
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param classIndex the attribute index of the class
     * @param missingLevel the percentage of missing values
     * @param predictorMissing true if the missing values may be in
     * the predictors
     * @param classMissing true if the missing values may be in the class
     * @param numTrain the number of instances in the training set
     * @param numTest the number of instaces in the test set
     * @param numClasses the number of classes
     * @param accepts the acceptable string in an exception
     * @return index 0 is true if the test was passed, index 1 is true if test
     *         was acceptable
     */
    protected boolean[] runBasicTest(boolean nominalPredictor,
        boolean numericPredictor,
        boolean stringPredictor,
        boolean datePredictor,
        boolean relationalPredictor,
        boolean multiInstance,
        int classType,
        int classIndex,
        int missingLevel,
        boolean predictorMissing,
        boolean classMissing,
        int numTrain,
        int numTest,
        int numClasses,
        FastVector accepts) {
  
      boolean[] result = new boolean[2];
      Instances train = null;
      Instances test = null;
      Classifier classifier = null;
      Evaluation evaluation = null;
      boolean built = false;
      try {
        train = makeTestDataset(42, numTrain,
                                nominalPredictor     ? getNumNominal()    : 0,
                                numericPredictor     ? getNumNumeric()    : 0,
                                stringPredictor      ? getNumString()     : 0,
                                datePredictor        ? getNumDate()       : 0,
                                relationalPredictor  ? getNumRelational() : 0,
                                numClasses,
                                classType,
                                classIndex,
                                multiInstance);
        test = makeTestDataset(24, numTest,
                               nominalPredictor     ? getNumNominal()    : 0,
                               numericPredictor     ? getNumNumeric()    : 0,
                               stringPredictor      ? getNumString()     : 0,
                               datePredictor        ? getNumDate()       : 0,
                               relationalPredictor  ? getNumRelational() : 0,
                               numClasses,
                               classType,
                               classIndex,
                               multiInstance);
        if (missingLevel > 0) {
          addMissing(train, missingLevel, predictorMissing, classMissing);
          addMissing(test, Math.min(missingLevel, 50), predictorMissing,
              classMissing);
        }
        classifier = AbstractClassifier.makeCopies(getClassifier(), 1)[0];
        evaluation = new Evaluation(train);
      } catch (Exception ex) {
        ex.printStackTrace();
        throw new Error("Error setting up for tests: " + ex.getMessage());
      }
      try {
        classifier.buildClassifier(train);
        built = true;
        if (!testWRTZeroR(classifier, evaluation, train, test)[0]) {
          result[0] = true;
          result[1] = true;
          throw new Exception("Scheme performs worse than ZeroR");
        }
  
        println("yes");
        result[0] = true;
      }
      catch (Exception ex) {
        boolean acceptable = false;
        String msg;
        if (ex.getMessage() == null)
          msg = "";
        else
          msg = ex.getMessage().toLowerCase();
        if (msg.indexOf("not in classpath") > -1)
          m_ClasspathProblems = true;
        if (msg.indexOf("worse than zeror") >= 0) {
          println("warning: performs worse than ZeroR");
          result[0] = true;
          result[1] = true;
        } else {
          for (int i = 0; i < accepts.size(); i++) {
            if (msg.indexOf((String)accepts.elementAt(i)) >= 0) {
              acceptable = true;
            }
          }
  
          println("no" + (acceptable ? " (OK error message)" : ""));
          result[1] = acceptable;
        }
  
        if (m_Debug) {
          println("\n=== Full Report ===");
          print("Problem during");
          if (built) {
            print(" testing");
          } else {
            print(" training");
          }
          println(": " + ex.getMessage() + "\n");
          if (!acceptable) {
            if (accepts.size() > 0) {
              print("Error message doesn't mention ");
              for (int i = 0; i < accepts.size(); i++) {
                if (i != 0) {
                  print(" or ");
                }
                print('"' + (String)accepts.elementAt(i) + '"');
              }
            }
            println("here are the datasets:\n");
            println("=== Train Dataset ===\n"
                + train.toString() + "\n");
            println("=== Test Dataset ===\n"
                + test.toString() + "\n\n");
          }
        }
      }
  
      return result;
    }
  
    /**
     * Determine whether the scheme performs worse than ZeroR during testing
     *
     * @param classifier the pre-trained classifier
     * @param evaluation the classifier evaluation object
     * @param train the training data
     * @param test the test data
     * @return index 0 is true if the scheme performs better than ZeroR
     * @throws Exception if there was a problem during the scheme's testing
     */
    protected boolean[] testWRTZeroR(Classifier classifier,
                                     Evaluation evaluation,
                                     Instances train, Instances test)
    throws Exception {
  
      boolean[] result = new boolean[2];
  
      evaluation.evaluateModel(classifier, test);
      try {
  
        // Tested OK, compare with ZeroR
        Classifier zeroR = new weka.classifiers.rules.ZeroR();
        zeroR.buildClassifier(train);
        Evaluation zeroREval = new Evaluation(train);
        zeroREval.evaluateModel(zeroR, test);
        result[0] = Utils.grOrEq(zeroREval.errorRate(), evaluation.errorRate());
      }
      catch (Exception ex) {
        throw new Error("Problem determining ZeroR performance: "
            + ex.getMessage());
      }
  
      return result;
    }
  
    /**
     * Make a simple set of instances, which can later be modified
     * for use in specific tests.
     *
     * @param seed the random number seed
     * @param numInstances the number of instances to generate
     * @param numNominal the number of nominal attributes
     * @param numNumeric the number of numeric attributes
     * @param numString the number of string attributes
     * @param numDate the number of date attributes
     * @param numRelational the number of relational attributes
     * @param numClasses the number of classes (if nominal class)
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param multiInstance whether the dataset should a multi-instance dataset
     * @return the test dataset
     * @throws Exception if the dataset couldn't be generated
     * @see #process(Instances)
     */
    protected Instances makeTestDataset(int seed, int numInstances,
                                        int numNominal, int numNumeric,
                                        int numString, int numDate,
                                        int numRelational,
                                        int numClasses, int classType,
                                        boolean multiInstance)
      throws Exception {
  
      return makeTestDataset(
          seed,
          numInstances,
          numNominal,
          numNumeric,
          numString,
          numDate,
          numRelational,
          numClasses,
          classType,
          TestInstances.CLASS_IS_LAST,
          multiInstance);
    }
  
    /**
     * Make a simple set of instances with variable position of the class
     * attribute, which can later be modified for use in specific tests.
     *
     * @param seed the random number seed
     * @param numInstances the number of instances to generate
     * @param numNominal the number of nominal attributes
     * @param numNumeric the number of numeric attributes
     * @param numString the number of string attributes
     * @param numDate the number of date attributes
     * @param numRelational the number of relational attributes
     * @param numClasses the number of classes (if nominal class)
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     * @param classIndex the index of the class (0-based, -1 as last)
     * @param multiInstance whether the dataset should a multi-instance dataset
     * @return the test dataset
     * @throws Exception if the dataset couldn't be generated
     * @see TestInstances#CLASS_IS_LAST
     * @see #process(Instances)
     */
    protected Instances makeTestDataset(int seed, int numInstances,
                                        int numNominal, int numNumeric,
                                        int numString, int numDate,
                                        int numRelational,
                                        int numClasses, int classType,
                                        int classIndex,
                                        boolean multiInstance)
    throws Exception {
  
      TestInstances dataset = new TestInstances();
  
      dataset.setSeed(seed);
      dataset.setNumInstances(numInstances);
      dataset.setNumNominal(numNominal);
      dataset.setNumNumeric(numNumeric);
      dataset.setNumString(numString);
      dataset.setNumDate(numDate);
      dataset.setNumRelational(numRelational);
      dataset.setNumClasses(numClasses);
      dataset.setClassType(classType);
      dataset.setClassIndex(classIndex);
      dataset.setNumClasses(numClasses);
      dataset.setMultiInstance(multiInstance);
      dataset.setWords(getWords());
      dataset.setWordSeparators(getWordSeparators());
  
      return process(dataset.generate());
    }
  
    /**
     * Print out a short summary string for the dataset characteristics
     *
     * @param nominalPredictor true if nominal predictor attributes are present
     * @param numericPredictor true if numeric predictor attributes are present
     * @param stringPredictor true if string predictor attributes are present
     * @param datePredictor true if date predictor attributes are present
     * @param relationalPredictor true if relational predictor attributes are present
     * @param multiInstance whether multi-instance is needed
     * @param classType the class type (NUMERIC, NOMINAL, etc.)
     */
    protected void printAttributeSummary(boolean nominalPredictor,
                                         boolean numericPredictor,
                                         boolean stringPredictor,
                                         boolean datePredictor,
                                         boolean relationalPredictor,
                                         boolean multiInstance,
                                         int classType) {
  
      String str = "";
  
      if (numericPredictor)
        str += " numeric";
  
      if (nominalPredictor) {
        if (str.length() > 0)
          str += " &";
        str += " nominal";
      }
  
      if (stringPredictor) {
        if (str.length() > 0)
          str += " &";
        str += " string";
      }
  
      if (datePredictor) {
        if (str.length() > 0)
          str += " &";
        str += " date";
      }
  
      if (relationalPredictor) {
        if (str.length() > 0)
          str += " &";
        str += " relational";
      }
  
      str += " predictors)";
  
      switch (classType) {
        case Attribute.NUMERIC:
          str = " (numeric class," + str;
          break;
        case Attribute.NOMINAL:
          str = " (nominal class," + str;
          break;
        case Attribute.STRING:
          str = " (string class," + str;
          break;
        case Attribute.DATE:
          str = " (date class," + str;
          break;
        case Attribute.RELATIONAL:
          str = " (relational class," + str;
          break;
      }
  
      print(str);
    }
  
    /**
     * Returns the revision string.
     *
     * @return		the revision
     */
    public String getRevision() {
      return RevisionUtils.extract("$Revision: 6041 $");
    }
  
    /**
     * Test method for this class
     *
     * @param args the commandline parameters
     */
    public static void main(String [] args) {
      runCheck(new CheckClassifier(), args);
    }
  }
  