content: /* The contents of this file are subject to the terms
   * of the Common Development and Distribution License
   * (the License). You may not use this file except in
   * compliance with the License.
   *
   * You can obtain a copy of the License at
   * http://www.sun.com/cddl/cddl.html or
   * install_dir/legal/LICENSE
   * See the License for the specific language governing
   * permission and limitations under the License.
   *
   * When distributing Covered Code, include this CDDL
   * Header Notice in each file and include the License file
   * at install_dir/legal/LICENSE.
   * If applicable, add the following below the CDDL Header,
   * with the fields enclosed by brackets [] replaced by
   * your own identifying information:
   * "Portions Copyrighted [year] [name of copyright owner]"
   *
   * $Id$
   *
   * Copyright 2005-2009 Sun Microsystems Inc. All Rights Reserved
   */
  package com.sun.faban.harness.engine;
  
  import com.sun.faban.common.*;
  import com.sun.faban.harness.FabanHostUnknownException;
  import com.sun.faban.harness.RemoteCallable;
  import com.sun.faban.harness.ParamRepository;
  import com.sun.faban.harness.ConfigurationException;
  import com.sun.faban.harness.agent.CmdAgent;
  import com.sun.faban.harness.agent.FileAgent;
  import com.sun.faban.harness.agent.FileService;
  import com.sun.faban.harness.common.Config;
  import com.sun.faban.harness.common.HostRoles;
  import com.sun.faban.harness.util.CmdMap;
  import com.sun.faban.harness.util.FileHelper;
  import com.sun.faban.harness.util.InterfaceProbe;
  
  import java.io.*;
  import java.net.*;
  import java.rmi.RemoteException;
  import java.text.SimpleDateFormat;
  import java.util.*;
  import java.util.concurrent.Callable;
  import java.util.concurrent.Future;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.TimeoutException;
  import java.util.logging.Level;
  import java.util.logging.Logger;
  
  /**
   * This file contains the class that implements the Command service API.
   * The Command Service object is created by the Engine at the start of
   * a run and it starts up the CmdAgent applications on all the
   * machines and connects to them via RMI. In the API implementation,
   * it identifies the particular CmdAgent and passes the call along.
   *
   * The CmdAgents take care of any error messages generated by the
   * command and automatically log them to the run's error log.
   * The CmdAgent's path will include the default faban bin
   * directories (in addition to /usr/bin, /usr/sbin, /usr/ucb), so
   * any faban executables will be found. Commands in any other
   * path should be invoked with the full pathname of the command.
   * The CmdAgent's environment will also include CLASSPATH set to
   * the faban lib directory to find any Java classes.
   *
   * Shell commands or any commands whose output must be re-directed
   * or piped (basically using shell) should be executed using syntax
   * such as :
   * "sh -c <command> [<args>] [> out]".
   * IMPORTANT: There should be a single CmdService object in the
   * entire framework or else multiple copies of the CmdAgent
   * application will be spawned on the target machines.
   * Therefore, this class is implemented as a Singleton.
   * No public constructors are defined and the object cannot be cloned.
   *
   * @author Ramesh Ramachandran
   * @see com.sun.faban.harness.agent.CmdAgent
   * @see com.sun.faban.harness.engine.GenericBenchmark
   */
  final public class CmdService { 	// The final keyword prevents clones
  
      /** Sequential flag in FG mode. */
      public static final int SEQUENTIAL = 1;
  
      /** Parallel flag in FG mode. */
      public static final int PARALLEL = 2;
  
      private static Logger logger = Logger.getLogger(CmdService.class.getName());
      private static CmdService cmds;
  
      private ArrayList<CmdAgent> cmdp = new ArrayList<CmdAgent>();
      private ArrayList<FileAgent> filep = new ArrayList<FileAgent>();
  
      /** List of all machines. */
      private ArrayList<String> machinesList = new ArrayList<String>();
      private Properties hostInterfaces = new Properties();
      private Registry registry;
      private String master;	// Name of faban master machine
      private String masterAddress; // ip of faban master machine
      private CommandHandle registryCmd;
      private String javaHome;
      private List<String> jvmOptions;
      private HashMap<String, List<String>> binMap =
              new HashMap<String, List<String>>();
      private Map<String, String> ifMap;
      private List<String> rsh,  agent;
      private HostRoles hostRoles;
  
      CmdService() {
  
          try {
              master = (InetAddress.getLocalHost()).getHostName();
              masterAddress = (InetAddress.getLocalHost()).getHostAddress();
              logger.config("InetAddress master Host = " + master);
              logger.config("InetAddress master address = " + masterAddress);
          } catch (Exception e) {
              logger.severe("CmdService <init> failed " + e);
              logger.log(Level.FINE, "Exception", e);
          }
          cmds = this;
      }
  
      /**
       * This method is the only way that an external object
       * can get a reference to the singleton CmdService.
       * This method should not be used outside engine.
       * @return reference to the single CmdService
       */
      public static CmdService getHandle() {
          return cmds;
      }
  
      /**
       * Obtains the name of the master machine.
       * @return The master machine name
       */
      public String getMaster() {
          return master;
      }
  
      /**
       * Returns the ip address of the master.
       * @return The ip address of the master
       */
      public String getMasterIP() {
          return masterAddress;
      }
  
      /**
       * Returns the ip address of the master's interface best used for
       * communicating with the target host.
       * @param agentHost The target host
       * @return The ip address of the master
       */
      public String getMasterIP(String agentHost) {
          return ifMap.get(agentHost);
      }
  
      /**
       *
       * This method is called after every run to re-initialize the data
       * structures that need to change from one run to another.
       *
       */
      public void init() {
          machinesList.clear();
          cmdp.clear();
          filep.clear();
          hostInterfaces.clear();
      }
  
      /**
       * This method initializes the CmdAgent RMI server processes
       * on the specified set of machines.
       * This method can be called multiple times to initialize multiple
       * classes of machines.
       * @param benchName The name of the benchmark
       * @param par The parameter repository
       * @return true if successful, false if setup failed
       */
      public boolean setup(String benchName, ParamRepository par) {
  
          String home = par.getParameter("fh:jvmConfig/fh:home");
  
          if (home != null)
              home = home.trim();
  
          if (home == null || home.length() == 0) {
              home = Utilities.getJavaHome();
              logger.config("JAVA_HOME set to " + home);
          }
  
          if(!(new File(home)).isDirectory()) {
              logger.severe("Cannot set JAVA_HOME. " + home +
                      " is not a valid JAVA_HOME. Exiting");
              return false;
          }
  
          javaHome = home;
  
          // Check whether the target JVM supports -XX:+DisableExplicitGC or not.
          String egc = "-XX:+DisableExplicitGC";
          Command probeCmd = new Command(javaHome + File.separator + "bin" +
                  File.separator + "java", egc, "-version");
          probeCmd.setLogLevel(Command.STDOUT, Level.FINER);
          probeCmd.setLogLevel(Command.STDERR, Level.FINER);
          try {
              CommandHandle handle = probeCmd.execute();
              if (handle.exitValue() != 0)
                  egc = null;
          } catch (IOException e) {
              egc = null;
          } catch (InterruptedException e) {
          }
  
          final String disableEGC = egc;
  
          // We need to be careful to escape properties having '\\' on win32
          String escapedHome = Config.FABAN_HOME.replace("\\", "\\\\");
          String fs = File.separatorChar == '\\' ? "\\\\" : File.separator;
  
          jvmOptions = new ArrayList<String>();
          jvmOptions.add("-Dfaban.home=" + escapedHome);
          jvmOptions.add("-Djava.security.policy=" + escapedHome + "config" +
                  fs + "faban.policy");
          jvmOptions.add("-Djava.util.logging.config.file=" + escapedHome +
                  "config" + fs + "logging.properties");
          jvmOptions.add("-Dfaban.registry.port=" + Config.RMI_PORT);
          jvmOptions.add("-Dfaban.logging.port=" + Config.LOGGING_PORT);
  
          try {
              // Update the logging.properties file in config dir
              Properties log = new Properties();
              FileInputStream in = new FileInputStream(Config.CONFIG_DIR +
                      "logging.properties");
              log.load(in);
              in.close();
  
              // Update if it has changed.
              if (!(log.getProperty("java.util.logging.SocketHandler.host").
                      equals(master) &&
                      log.getProperty("java.util.logging.SocketHandler.port").
                      equals(String.valueOf(Config.LOGGING_PORT)))) {
                  log.setProperty("java.util.logging.SocketHandler.host", master);
                  log.setProperty("java.util.logging.SocketHandler.port",
                          String.valueOf(Config.LOGGING_PORT));
                  FileOutputStream out = new FileOutputStream(
                          new File(Config.CONFIG_DIR + "logging.properties"));
                  log.store(out, "Faban logging properties");
                  out.close();
              }
  
          } catch (IOException e) {
              logger.log(Level.SEVERE, "Failed to initialize CmdAgent " + e, e);
          }
  
          // Start RMI registry and Registry
          try {
  
              // Create classpath with all client jars in faban/lib dir.
              // Benchmark specific stubs will be in one of the jars.
              File[] libs = (new File(Config.LIB_DIR)).listFiles();
  
              StringBuffer buf = new StringBuffer();
              for (int i = 0; i < libs.length; i++) {
                  if (libs[i].isFile()) {
                      buf.append(libs[i].getAbsolutePath() + File.pathSeparator);
                  }
              }
              buf.setLength(buf.length() - 1);
              if (buf.indexOf(" ") != -1) {
                  buf.insert(0, '"');
                  buf.append('"');
              }
              String classpath = buf.toString();
  
              // The registry should not consume much resources. Just don't
              // use the driver JVM options and set it to 32m - 1024m dynamic.
              // This should not be performance sensitive at all.
              List<String> cmd = new ArrayList<String>();
              cmd.add(javaHome + File.separator + "bin" + File.separator +
                      "java");
              cmd.addAll(jvmOptions);
              cmd.add("-Xms32m");
              cmd.add("-Xmx1024m");
              cmd.add("-cp");
              cmd.add(classpath);
              cmd.add("com.sun.faban.common.RegistryImpl");
  
              logger.info("Starting Registry.");
              Command rmiCmd = new Command(cmd);
              rmiCmd.setSynchronous(false);
              rmiCmd.setLogLevel(Command.STDOUT, Level.WARNING);
              registryCmd = rmiCmd.execute();
  
          } catch (Exception e) {
              logger.log(Level.SEVERE, "Couldn't start Registry. " +
                      "Please check if its already running", e);
              return false;
          }
  
          // Now add the driver options to the JVM options. Need them after this.
          String jvmOpts =
                  par.getParameter("fh:jvmConfig/fh:jvmOptions");
  
          if (jvmOpts != null)
              jvmOpts = jvmOpts.trim();
  
          if((jvmOpts == null) || (jvmOpts.length() == 0))
              jvmOpts = "";
  
          List<String> usrOpts = Command.parseArgs(jvmOpts);
          if (disableEGC != null && !usrOpts.contains(disableEGC))
              usrOpts.add(disableEGC);
          jvmOptions.addAll(usrOpts);
  
          // RMI registry takes a bit of time to startup. So sleep for some time
          try {
              logger.fine("Waiting for RMI registry and Registry to startup");
              Thread.sleep(10000);
          } catch (InterruptedException e) {
          }
  
          try {
              registry = RegistryLocator.getRegistry(Config.RMI_PORT);
          } catch (Exception e) {
              logger.log(Level.SEVERE, "Unable to connect to Registry.", e);
              return false;
          }
  
          // an agent needs to be started on the master machine
          // first since configuration of agents on other machines
          // depend on a CmdAgent running on the master machine
  
          // We need to scan the machines to ensure that they are not a different
          // incarnation of the master's name. If they are, switch the master to
          // use these names instead.
  
          // Also, we use the same loop to create a non-duplicate set of remote
          // machines. This is used later to find the interfaces to the remote
          // machine.
  
          InetAddress[] masterIps = null;
          try {
              masterIps = InetAddress.getAllByName(master);
          } catch (UnknownHostException e) {
              logger.log(Level.SEVERE, "Strange! Master is unknown.", e);
              return false;
          }
  
          HashSet<String> remoteMachines = new HashSet<String>();
          boolean isMasterSet = false;
  
          List<ParamRepository.HostConfig> hostConfigs = null;
          try {
              hostConfigs = par.getHostConfigs();
          } catch (ConfigurationException e) {
              logger.log(Level.SEVERE, "Problem reading parameter file", e);
          }
  
          outer:
          for (ParamRepository.HostConfig hostConfig : hostConfigs) {
              String[] machines = hostConfig.hosts;
              for (int i = 0; i < machines.length; i++) {
  
                  // Check for no localhost, we don't allow it.
                  if (machines[i].startsWith("localhost")) {
                      if (machines[i].length() == 9 || // localhost
                              machines[i].charAt(9) == '.') { // localhost.domain
                          logger.severe("Host names must not be localhost. " +
                                  "Please use real host names or IP addresses " +
                                  "instead. Terminating run!");
                          return false;
                      }
                  }
                  try {
                      InetAddress[] machineIps =
                              InetAddress.getAllByName(machines[i]);
                      if (sameHost(masterIps, machineIps)) {
                          if (!isMasterSet) { // Set the master to the first
                              // found master name in the list.
                              master = machines[i];
                              isMasterSet = true;
                          } else { // Set all subsequent masters to the same.
                              machines[i] = master;
                          }
                      } else {     // All remote machines go into a set.
                          remoteMachines.add(machines[i]);
                      }
                  } catch (UnknownHostException e) {
                      logger.log(Level.WARNING, machines[i] + " is unknown.", e);
                  }
              }
          }
  
          // Next we use the command map to get the right
          // rsh command based on the undelying OS.
          try {
              binMap = CmdMap.getCmdMap(null);
              rsh = binMap.get("rsh");
              agent = binMap.get("agent");
          } catch (Exception e) {
              logger.log(Level.WARNING, "Failed to obtain command map.", e);
          }
  
          if (rsh == null) {
              rsh = new ArrayList<String>();
              rsh.add("rsh");
          }
  
          //only case in which interfaceAddress is not an address but
          //the hostname of the master machine.  used in CmdAgentImpl
          //the cmdagent on the master machine is registered under 2
          // names, Config.CMD_AGENT@master as well as just Config.CMD_AGENT
          if (!machinesList.contains(master)) {
              if (!startCmdAgent(benchName, master, master)) {
                  return false;
              }
              machinesList.add(master);
          }
  
          // this is necessary in case you are on a private network
          // where the machine's private ip address is not the same as it's
          // public ip address
  
          // Fist check specific scripts for the arch
          String scriptPath = Config.BIN_DIR + Config.ARCH_DIR + "interface";
          File ifScript = new File(scriptPath.trim());
  
          // Then check script for the OS. If it exists, use it.
          // It is usually more reliable than the interface probe.
          if (!ifScript.exists()) {
              logger.finer("Could not find interface script at " +
                      ifScript.getAbsolutePath());
              scriptPath = Config.BIN_DIR + Config.OS_DIR + "interface";
              ifScript = new File(scriptPath.trim());
          }
  
          ifMap = new HashMap<String, String>();
          boolean ifMapComplete = false;
  
          if (ifScript.exists()) {
              ifMapComplete = getIfMap(remoteMachines, ifScript, ifMap);
          } else {
              logger.finer("Could not find interface script at " +
                      ifScript.getAbsolutePath());
              ifScript = null;
          }
  
          // If we have no interface script or the interface script did not
          // do a complete job, we'll resort to the probe.
          // Most reliable when run as root, but buggy in parallel mode.
          // Also the interface probe needs JDK1.6 or later.
          if (!ifMapComplete) {
              if ("1.6".compareTo(System.getProperty("java.version")) > 0) {
                  logger.severe("Could not find a way to check the interface!");
                  return false;
              }
  
              InterfaceProbe iProbe = null;
              try {
                  iProbe = new InterfaceProbe(Config.THREADPOOL);
                  iProbe.getIfMap(remoteMachines, ifMap);
              } catch (SocketException e) {
                  logger.log(Level.SEVERE,
                          "Could not find a way to check the interface!", e);
              }
          }
  
  
          // cycles through benchmark machines starting up agents and
          // configuring them
          for (ParamRepository.HostConfig hostConfig : hostConfigs) {
              String[] machines = hostConfig.hosts;
              for (int i = 0; i < machines.length; i++) {
                  // Do not start duplicate Cmd agent
                  if (machinesList.contains(machines[i])) {
                      continue;
                  }
  
                  String interfaceAddress = ifMap.get(machines[i]);
  
                  if (interfaceAddress == null || interfaceAddress.length() == 0) {
                      return false;
                  }
  
                  if (!startCmdAgent(benchName, machines[i], interfaceAddress)) {
                      return false;
                  }
  
                  // By adding the mach to the list we prevent multiple
                  // agents being started on the same server
                  machinesList.add(machines[i]);
              }
          }
          try {
              Thread.sleep(20000);
          } catch (InterruptedException e) {
          }
          for (int i = 0; i < machinesList.size(); i++) {
              if (!getCmdAgent((String) machinesList.get(i))) {
                  return false;
              }
          }
  
          if (par.getBooleanValue("fa:runConfig/fh:timeSync", true)) {
              setClocks();
          }
  
          return true;
      }
  
      void setHostRoles(HostRoles hr) {
          hostRoles = hr;
  
          // We need to populate the machinesList and cmdp with
          // the real host names.
  
          // First get the real names.
          String[] realHosts = hostRoles.getHostsInOrder();
          
          // For each real host name not in machinesList, we take the first alias
          // and look it up in the machinesList. Fetch the command agent and add
          // the real name to the machinesList -> cmdp mapping.
          for (String hostName : realHosts) {
              if (!machinesList.contains(hostName)) {
                  String[] aliases = hostRoles.getAliasesByHost(hostName);
                  CmdAgent a = findCmdAgent(aliases[0]); // Just one is enough
                  machinesList.add(hostName);
                  cmdp.add(a);
              }
          }
      }
  
      private boolean getIfMap(Collection<String> hosts, File ifScript,
              Map<String, String> ifMap) {
          boolean complete = true;
  
          for (String host : hosts) {
              String interfaceAddress = null;
  
              String ifCommand = ifScript.getAbsolutePath() + ' ' + host;
  
              logger.fine("Detecting interface: " + ifCommand);
              try {
                  Process p = Runtime.getRuntime().exec(ifCommand);
                  BufferedReader bufR = new BufferedReader(
                          new InputStreamReader(p.getInputStream()));
  
                  interfaceAddress = bufR.readLine();
                  if (interfaceAddress != null) {
                      interfaceAddress = interfaceAddress.trim();
                      if ("127.0.0.1".equals(interfaceAddress)) {
                          complete = false;
                          ifMap.put(host, "");
                      } else {
                          ifMap.put(host, interfaceAddress);
                      }
                  }
  
                  int exitValue = -1;
  
                  if (interfaceAddress != null &&
                          interfaceAddress.length() > 0) { //Read something...
  
                      exitValue = p.waitFor();
                      if (exitValue != 0) {
                          logger.warning("interface: Cannot reach system " +
                                  host);
                          complete = false;
                          ifMap.put(host, "");
                          continue;
                      }
                  } else { // Nothing read, check stderr
                      bufR = new BufferedReader(
                              new InputStreamReader(p.getErrorStream()));
                      logger.severe(host + ": " + bufR.readLine());
                      ifMap.put(host, "");
                      continue;
                  }
              } catch (Exception e) {
                  logger.log(Level.SEVERE,
                          "Error in executing the interface program: " +
                          ifCommand, e);
                  break;
              }
  
              logger.config("Interface Address = " + interfaceAddress);
          }
          return complete;
      }
  
      private boolean getCmdAgent(String mach) {
  
          try {
              String s = Config.CMD_AGENT + "@" + mach;
              logger.fine("CmdService: Connecting to " + s);
              int retry = 1;
              CmdAgent c = (CmdAgent) registry.getService(s);
              for (; c == null && retry <= 10; retry++) {
                  Thread.sleep(10000);
                  logger.warning("Retry connecting to " + s + ", count " +
                          retry + '.');
                  c = (CmdAgent) registry.getService(s);
              }
              if (c == null) {
                  logger.severe("Could not connect to " + s);
                  return (false);
              }
  
              cmdp.add(c);
  
              /* Note the agent registration process:
               * 1. Create and register the command agent.
               * 2. Download benchmark code
               * 3. Create the lib classpath
               * 4. Create and register file agent
               * So it may take quite some time between the registration of
               * the command agent and the file agent. But we can be pretty
               * sure it'll happen. So just wait. Timeout after 100 retries.             
               */
              s = Config.FILE_AGENT + "@" + mach;
              logger.fine("FileService: Connecting to " + s);
              retry = 1;
              FileAgent f = (FileAgent) registry.getService(s);
              for (; f == null && retry <= 100; retry++) {
                  Thread.sleep(1000);
                  logger.fine("Retry obtaining file service from " + s +
                          ", count " + retry + '.');
                  f = (FileAgent) registry.getService(s);
              }
              if (f == null) {
                  logger.severe("Timed out obtaining file service from " + s);
                  return (false);
              }
              filep.add(f);
  
              // Added by Ramesh to get the real hostnames of the servers
              logger.info("CmdService: Configured " + s + " on server " +
                      c.getHostName());
              return true;
  
          } catch (Exception e) {
              logger.log(Level.SEVERE, "Error accessing command agent on system " + mach, e);
              return (false);
          }
      }
  
      /* start up the CmdAgent applications
       * We use a script 'cmd' which will setup the CLASSPATH before
       * invoking CmdAgent
       */
      private boolean startCmdAgent(String benchName, String mach,
              String interfaceAddress) {
  
          hostInterfaces.setProperty(mach, interfaceAddress);
          List<String> cmd = new ArrayList<String>();
  
          List<String> agentParams = new ArrayList<String>();
          agentParams.add(mach);
          agentParams.add(interfaceAddress);
          agentParams.add(masterAddress);
          agentParams.add(javaHome);
          agentParams.addAll(jvmOptions);
          agentParams.add("faban.benchmarkName=" + benchName);
          try {
              if (mach.equals(master)) {
                  cmd.addAll(agent);
                  cmd.addAll(agentParams);
                  logger.fine("Executing " + cmd);
                  Command cmdAgent = new Command(cmd);
                  cmdAgent.setSynchronous(false);
                  cmdAgent.setLogLevel(Command.STDOUT, Level.WARNING);
                  cmdAgent.execute();
              } else { // if the machine is not the master machine, we need to
                  // do an rsh or talk to the agent daemon and pass download
                  // instructions.
                  // Many times, the FABAN_URL cannot be reached by the benchmark
                  // downloader. So it is better to change the URL to access
                  // the master via the best interface, by ip address instead of
                  // host name.
  
                  URL fabanURL = new URL(Config.FABAN_URL);
                  URL downloadURL = new URL(fabanURL.getProtocol(),
                          interfaceAddress, fabanURL.getPort(),
                          fabanURL.getFile());
                  agentParams.add("faban.download=" + downloadURL.toString());
  
                  boolean agentStarted = false;
  
                  try { // See first whether we have an agent daemon.
                      Socket socket = new Socket(mach, Config.AGENT_PORT);
                      ObjectOutputStream socketOut =
                              new ObjectOutputStream(socket.getOutputStream());
                      InputStream socketIn = socket.getInputStream();
                      byte[] buffer = new byte[1024];
  
                      ArrayList<String> agentExtParams = 
                              new ArrayList<String>(agentParams);
                      agentExtParams.add(File.separator);
                      agentExtParams.add(File.pathSeparator);
                      socketOut.writeObject(agentExtParams);
  
                      int length = socketIn.read(buffer);
                      socketIn.close();
                      socketOut.close();
                      socket.close();
                      String response = new String(buffer, 0, length);
                      int rcode = Integer.parseInt(response.substring(0, 3));
                      switch (rcode) {
                          case 200:
                              agentStarted = true;
                              logger.fine("Found Agent(daemon)@" + mach +
                                      ". Registering agent.");
                              break;
                          case 500:
                              logger.warning("Agent(daemon)@" + mach +
                                      ": " + response +
                                      " Please report the issue " +
                                      "and provide logs from " + mach +
                                      ":FABAN_HOME/logs/agent.log");
                              break;
                          case 409:
                              logger.severe("Agent(daemon)@" + mach +
                                      ": " + response);
                              // We do not fall back in the conflict case.
                              return false;
                          default:
                              logger.warning("Agent(daemon)@" + mach +
                                      ": " + response);
                      }
  
                  } catch (ConnectException e) {
                      // We should get a ConnectException if the agent was not
                      // started in daemon mode. This should take no time.
                      logger.log(Level.FINER, "Agent(daemon)@" + mach + ": " +
                              e.getMessage() + ". Will try remote shell instead.", e);
                  } catch (IOException e) {
                      logger.log(Level.WARNING, "Agent(daemon)@" + mach + ": " +
                              e.getMessage() + ". Will try remote shell instead.", e);
                  }
  
                  if (!agentStarted) {
                      cmd.clear();
                      cmd.addAll(rsh);
                      cmd.add(mach);
                      cmd.addAll(agent);
                      cmd.addAll(agentParams);
                      Command cmdAgent = new Command(cmd);
                      cmdAgent.setSynchronous(false);
                      cmdAgent.setLogLevel(Command.STDOUT, Level.WARNING);
                      cmdAgent.execute();
                  }
              }
              return true;
          } catch (Exception e) {
              logger.log(Level.SEVERE, "Could not execute " + agent +
                      "on machine " + mach, e);
              return false;
          }
      }
  
      private boolean sameHost(InetAddress[] host1, InetAddress[] host2) {
          for (int i = 0; i < host1.length; i++) {
              for (int j = 0; j < host2.length; j++) {
                  if (host1[i].equals(host2[j])) {
                      return true;
                  }
              }
          }
          return false;
      }
  
      private void setClocks() {
          SimpleDateFormat dateFormat = new SimpleDateFormat("MMddHHmmyyyy.ss");
          dateFormat.setTimeZone(new SimpleTimeZone(0, "GMT")); // Use GMT.
          HashSet<String> hostSet = new HashSet<String>();
          ArrayList<NameValuePair<Future<Boolean>>> tasks =
                  new ArrayList<NameValuePair<Future<Boolean>>>();
          hostSet.add(master); // Don't try to set clock for master.
          for (Object o : cmdp) {
              CmdAgent agent = (CmdAgent) o;
              String hostName = null;
              try {
                  hostName = agent.getHostName();
                  if (hostSet.add(hostName)) {
                      NameValuePair<Future<Boolean>> future =
                              new NameValuePair<Future<Boolean>>();
                      future.name = hostName;
                      future.value = Config.THREADPOOL.submit(
                              new setClockTask(agent, hostName, dateFormat));
                      tasks.add(future);
                  }
              } catch (RemoteException e) {
                  logger.log(Level.WARNING,
                          "Cannot communicate to agent to set time.", e);
              }
          }
          for (NameValuePair<Future<Boolean>> future : tasks) {
              try {
                  future.value.get(300, TimeUnit.SECONDS);
              } catch (TimeoutException e) {
                  logger.log(Level.WARNING, "Timed out setting clock for " +
                          future.name);
              } catch (Throwable t) {
                  Throwable cause = t.getCause();
                  while (cause != null) {
                      t = cause;
                      cause = t.getCause();
                  }
                  logger.log(Level.WARNING, t.getMessage(), t);
              }
          }
      }
  
      static class setClockTask implements Callable<Boolean> {
  
          public static final long ACCURACY = 10l; // plus-minus 10ms.
          CmdAgent agent;
          String hostName;
          SimpleDateFormat dateFormat;
  
          setClockTask(CmdAgent agent, String hostName,
                  SimpleDateFormat dateFormat) {
              this.agent = agent;
              this.hostName = hostName;
              this.dateFormat = dateFormat;
          }
  
          public Boolean call() throws IOException, InterruptedException {
  
              // 1. If we're within accuracy, don't set the clock
              long ms = System.currentTimeMillis();
              long timeDiff = -agent.getTime() +
                      ms + (System.currentTimeMillis() - ms) / 2;
              if (timeDiff < ACCURACY && timeDiff > -ACCURACY) {
                  logger.fine("Time difference of " + timeDiff +
                          " ms already in range. No need to set clock.");
                  return true;
              }
  
              logger.info("Time difference to host " + hostName + " is " +
                      timeDiff + " ms. Attempting to set clock.");
  
              int lag = 100; // Start with 100ms latency.
              int wakeBefore = 20;
  
              // 2. Wait till we're latency/2 from second boundary
              // Find next second boundary.
              long nextSec;
              String nextSecString = "";
              long callTime;
  
              for (int i = 0;; i++) {
                  if (i >= 20) {
                      logger.warning(hostName + " cannot accurately set remote " +
                              "time after " + i + " attempts. There is still a " +
                              "difference of " + timeDiff + " ms. Giving up.");
                      return false;
                  }
                  findBoundaryLoop:
                  for (int j = 0;; j++) {
                      if (j >= 20) {
                          logger.warning(hostName + " cannot scan time to set " +
                                  "clock after " + j + " retries. Giving up " +
                                  "setting clock. System may be overloaded or " +
                                  "JVM doing too much garbage collections.");
                          return false;
                      }
                      logger.finer("Lag time: " + lag + "ms");
                      for (;;) {
                          ms = System.currentTimeMillis();
                          nextSec = (long) Math.ceil(ms / 1000d);
                          // We should be 100 ms from the boundary, at least.
                          if (nextSec * 1000 - ms < 100) {
                              ++nextSec; // If not, we go to the next sec.
                          }
                          // Convert nextSec back to millis
                          nextSec *= 1000l;
                          callTime = nextSec - lag;
  
                          // DateFormat got passed to us and gets shared between
                          // multiple threads. So we need to sync.
                          synchronized (dateFormat) {
                              nextSecString = dateFormat.format(
                                      new Date(nextSec));
                          }
  
                          // Now, sleep and wake up 20ms before the wanted second
                          // boundary. This is to avoid late calls as sleep may
                          // have up to 10ms wakeup delay.
                          long sleepTime = callTime - wakeBefore -
                                  System.currentTimeMillis();
                          if (sleepTime > 0) {
                              Thread.sleep(sleepTime);
                          }
  
                          if (System.currentTimeMillis() >= callTime - 2) {
                              wakeBefore += wakeBefore;
                              if (wakeBefore > 700) {
                                  logger.warning(hostName + " wakeup-before " +
                                          "time reached 700ms limit. System is " +
                                          "too busy. Giving up.");
                                  return false;
                              }
                              continue;
                          }
                          break;
                      }
  
                      // Now within 20ms from the call, wait in a tight loop.
                      for (;;) {
                          long currentTime = System.currentTimeMillis();
                          if (currentTime == callTime) {
                              break findBoundaryLoop;
                          } else if (currentTime > callTime) {
                              logger.finer(hostName + "missed preset callTime " +
                                      "of " + callTime + ". Current time is " +
                                      currentTime + ".");
                              continue findBoundaryLoop; // Missed second boundary
                          }
                      }
                  }
  
                  // 3. Call agent to set time
                  ms = System.currentTimeMillis();
                  agent.setTime(nextSecString);
                  logger.finer("Actual setTime took " +
                          (System.currentTimeMillis() - ms) + " ms.");
  
                  // 4. Verify that time has been set properly.
                  ms = System.currentTimeMillis();
                  timeDiff = -agent.getTime() +
                          ms + (System.currentTimeMillis() - ms) / 2;
                  if (timeDiff < ACCURACY && timeDiff > -ACCURACY) {
                      logger.info("Setting time succeeded for " + hostName +
                              " after " + i + " retries. Time difference is " +
                              timeDiff + " ms.");
                      break;
                  } else {
                      logger.finer("Too large time difference of " + timeDiff +
                              " ms to " + hostName + ". Only " + ACCURACY +
                              " ms are allowed.");
                      lag += timeDiff;
                  }
              }
              return true;
          }
      }
  
      /**
       * Obtains the cached HostType object. Note that this is not a public API.
       * @return The cached HostType object;
       */
      public HostRoles getHostRoles() {
          return hostRoles;
      }
  
      /**
       * Returns the hostname of this machine as known to the machine
       * itself. This method is included in order to solve a Naming problem
       * related to the names of the tpcw result files to be transferred to the
       * the master machine.
       * @param machineName The target machine to check the host name
       * @return The host name of the remote machine
       */
      public String getHostName(String machineName) {
  
          int index = machinesList.indexOf(machineName);
          if (index < 0) {
              return machineName; // Cannot resolve
          }
          String retVal = null;
          try {
              retVal = cmdp.get(index).getHostName();
          } catch (RemoteException re) {
              logger.severe("RemoteException " +
                      re.getCause());
              logger.log(Level.FINE, "Exception", re);
          }
          if (retVal == null) {
              return machineName;
          }
          return retVal;
      }
  
      /**
       * Updates the paths in the local command agent.
       * @param pathList The list of paths to download
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public void updatePaths(List<String> pathList)
              throws RemoteException {
          findCmdAgent(master).updatePaths(pathList);
      }
  
      /**
       * Downloads files used by deploy images, especially services and tools.
       * The pathList contains a list of resources in the form type/resource.
       * @param machine The host name to initiate the download
       * @param pathList The list of paths to download
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public void downloadServices(String machine, List<String> pathList)
              throws RemoteException {
          findCmdAgent(machine).downloadServices(pathList);
      }
  
      /**
       * Returns the location of this command on the master system.
       * Similar to the which shell command, 'which' returns the actual path
       * to the given command. If it maps to a series of commands, they will
       * be returned as a single string separated by spaces. Note that 'which'
       * does not actually try to check the underlying system for commands
       * in the search path. It only checks the Faban infrastructure for
       * existence of such a command.
       * @param cmd The command to search for
       * @param svcPath The service path, if any
       * @return The actual command to execute, or null if not found.
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public String which(String cmd, String svcPath) throws RemoteException {
          return which(master, cmd, svcPath);
      }
  
      /**
       * Returns the location of this command on the target system.
       * Similar to the which shell command, 'which' returns the actual path
       * to the given command. If it maps to a series of commands, they will
       * be returned as a single string separated by spaces. Note that 'which'
       * does not actually try to check the underlying system for commands
       * in the search path. It only checks the Faban infrastructure for
       * existence of such a command.
       * @param machine The machine to search
       * @param cmd The command to search for
       * @param svcPath The service path, if any
       * @return The actual command to execute, or null if not found.
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public String which(String machine, String cmd, String svcPath)
              throws RemoteException {
          return findCmdAgent(machine).which(cmd, svcPath);
      }
  
      /**
       * Returns the location of this command on the target systems.
       * Similar to the which shell command, 'which' returns the actual path
       * to the given command. If it maps to a series of commands, they will
       * be returned as a single string separated by spaces. Note that 'which'
       * does not actually try to check the underlying system for commands
       * in the search path. It only checks the Faban infrastructure for
       * existence of such a command.
       * @param machines The machines to search
       * @param cmd The command to search for
       * @param svcPath The service path, if any
       * @return The actual command paths to execute, or null elements if not found.
       */
      public String[] which(String[] machines, String cmd, String svcPath) {
          String[] paths = new String[machines.length];
          for (int i = 0; i < machines.length; i++)
              try {
                  paths[i] = which(machines[i], cmd, svcPath);
              } catch (RemoteException e) {
                  logger.warning("Error searching for command " + cmd + " on " +
                                  machines[i] + '.');
              }
          return paths;
      }
  
      /**
       * Executes a command from the master's command agent.
       * @param c The command to be executed
       * @param svcPath
       * @return  A handle to the command
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle execute(Command c, String svcPath)
              throws IOException, InterruptedException {
          return execute(master, c, svcPath);
      }
  
      /**
       * Executes a command from the remote command agent.
       * @param machine The target machine to execute the command
       * @param c The command to be executed
       * @param svcPath The location of the invoking service, if any
       * @return  A handle to the command
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle execute(String machine, Command c, String svcPath)
              throws IOException, InterruptedException {
          return findCmdAgent(machine).execute(c, svcPath);
      }
  
      /**
       * Executes a command from the remote command agent.
       * @param machines The target machines to execute the command
       * @param c The command to be executed
       * @param svcPath The location of the invoking service, if any
       * @return  Handles to the command on each of the target machines
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle[] execute(String[] machines, Command c, String svcPath)
              throws IOException, InterruptedException {
          CommandHandle[] result = new CommandHandle[machines.length];
          for (int i = 0; i < machines.length; i++)
              result[i] = findCmdAgent(machines[i]).execute(c, svcPath);
          return result;
      }
  
      /**
       * Executes a java command from the master's command agent.
       * @param c The command to be executed
       * @param svcPath The location of the invoking service, if any
       * @return  A handle to the command
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle java(Command c, String svcPath)
              throws IOException, InterruptedException {
          return java(master, c, svcPath);
      }
  
      /**
       * Executes a java command from the remote command agent.
       * @param machine The target machine to execute the command
       * @param c The command to be executed
       * @param svcPath The location of the invoking service, if any
       * @return  A handle to the command
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle java(String machine, Command c, String svcPath)
              throws IOException, InterruptedException {
          return findCmdAgent(machine).java(c, svcPath);
      }
  
      /**
       * Executes a java command from the remote command agent.
       * @param machines The target machines to execute the command
       * @param c The command to be executed
       * @param svcPath The location of the invoking service, if any
       * @return  Handles to the command on each of the target machines
       * @throws IOException Error communicating with resulting process
       * @throws InterruptedException Thread got interrupted waiting
       * @throws RemoteException If there is a communication error to the
       *                         remote agent
       */
      public CommandHandle[] java(String[] machines, Command c, String svcPath)
              throws IOException, InterruptedException {
          CommandHandle[] result = new CommandHandle[machines.length];
          for (int i = 0; i < machines.length; i++)
              result[i] = findCmdAgent(machines[i]).java(c, svcPath);
          return result;
      }
  
      /**
       * Executes a job in a remote command agent.
       * @param machine The host to execute the command
       * @param callable The job
       * @param svcPath The location of the invoking service, if any
       * @return The return value of the job
       * @throws Exception An error occured executing the remote job
       */
      public <V extends Serializable> V execute(String machine,
                                                RemoteCallable<V> callable,
                                                String svcPath)
              throws Exception {
          return findCmdAgent(machine).exec(callable, svcPath);
      }
  
      /**
       * Executes a job on remote command agents on a list of systems.
       * @param machines The host names to execute the job
       * @param callable The job
       * @param svcPath The location of the invoking service, if any
       * @return The return values of the job, in sequence
       * @throws Exception An error occurred executing the job
       */
      public <V extends Serializable> List<V> execute(String[] machines,
                                                      RemoteCallable<V> callable,
                                                      String svcPath)
              throws Exception {
  
          ArrayList<V> rl = new ArrayList<V>();
          for (int i = 0; i < machines.length; i++)
              rl.add(findCmdAgent(machines[i]).exec(callable, svcPath));
          return rl;
      }
  
      /**
       * Start the agent on a single machine.
       * @param machine on which command should be started
       * @param agentClass Impl Class of the agent to be started
       * @param identifier to identify this agent later
       * @return true if the command completed successfully, else false
       * @throws Exception An error occurred starting the command
       */
      public boolean startAgent(String machine, Class agentClass,
                                String identifier) throws Exception {
  
          String m[] = new String[1];
          m[0] = machine;
  
          return (startAgent(m, agentClass, identifier));
      }
  
      /**
       * Start Agent in the specified machines.
       *
       * @param machines on which command should be started
       * @param agentClass Impl Class of the agent to be started
       * @param identifier to identify this agent later
       * @return true if all commands completed successfully, else false
       * @throws Exception An error occurred starting the commands
       */
      public boolean startAgent(String machines[], Class agentClass,
                                String identifier) throws Exception {
          boolean result = true;
  
          for (int i = 0; i < machines.length; i++) {
              if ((machines[i] == null) || (machines[i].equals(""))) {
                  continue;
              }
              //Change the identifier to agent@host
              result = result && findCmdAgent(machines[i]).
                      startAgent(agentClass, identifier + "@" + machines[i]);
          }
          return result;
      }
  
      /**
       * Gets a property from a given file.
       * @param machine The machine name
       * @param propFile The property file name
       * @param propName The property key name
       * @return The property value
       * @throws java.io.IOException If there is an error accessing the config file
       */
      public String getProperty(String machine, String propFile, String propName)
              throws IOException {
          return findFileAgent(machine).getProperty(propFile, propName);
      }
  
      /**
       * Kill all commands currently running and cleanup.
       * This method is called when a run must be aborted
       * or at the end of a benchmark run.
       */
      public void kill() {
          int i = 0;
          try {
              for (i = 0; i < cmdp.size(); i++) {
                  logger.info("killing CmdAgent@" + machinesList.get(i));
                  cmdp.get(i).kill();
              }
              cmdp.clear();
              filep.clear();
          } catch (Exception e) {
              logger.log(Level.SEVERE, "Kill Failed for CmdAgent@" +
                      machinesList.get(i), e);
          } finally {
              //Exiting Registry
              if (registryCmd != null) {
                  int retry = 0;
                  for (; retry < 20; retry++) {
                      try {
                          registryCmd.destroy();
                          Thread.sleep(1000);
                          int exitValue = registryCmd.exitValue();
                          logger.finer("Registry exited with exit value " +
                                  exitValue + '.');
                          break;
                      } catch (InterruptedException e) {
                          logger.log(Level.WARNING, "Interrupted waiting for " +
                                  "registry to terminate. " +
                                  "Cannot verify termination status.", e);
                      } catch (RemoteException e) {
                          logger.log(Level.SEVERE, "Caught RemoteException on " +
                                  "local CommandHandle destroy for Registry. " +
                                  "Please report bug.", e);
                      } catch (IllegalThreadStateException e) {
                          logger.log(Level.FINER,
                                  "Registry did not terminate! ", e);
                      }
                  }
                  if (retry == 20) {
                      logger.severe("Registry did not terminate " +
                              "after 20 termination attempts, giving up! " +
                              "Subsequent runs may have problems.");
                  }
  
              }
          }
      }
  
      /**
       * Pushes a local file on the Faban master to the remote host.
       * @param srcfile The source file name, relative to the out dir
       * @param destmachine The destination machine
       * @param destfile The destination file name
       * @return true if successful, false otherwise
       */
      public synchronized boolean push(String srcfile,
              String destmachine, String destfile) {
          int didx = machinesList.indexOf(destmachine);
          if (didx == -1) {
              throw new FabanHostUnknownException(
                      "Host " + destmachine + " not found!");
          }
          
          // Ensure the file is accessed from the right place.
          File src = new File(srcfile);
          if (!src.isAbsolute())
              src = new File(Config.OUT_DIR, srcfile);
          srcfile = src.getAbsolutePath();
  
          try {
              String srcHost = InetAddress.getLocalHost().getHostName();
              String destHost = hostRoles.getHostByAlias(destmachine);
              if (destHost.equals(srcHost)) {
                  if (srcfile.equals(destfile)) {
                      return true;
                  } else {
                      return FileHelper.copyFile(srcfile, destfile, true);
                  }
              }
          } catch (UnknownHostException e) {
              logger.log(Level.SEVERE, "CmdService: Cannot determine own" +
                      "host name", e);
              return false;
          }
  
          FileAgent destf = filep.get(didx);
          try {
              FileTransfer transfer = new FileTransfer(srcfile, destfile);
              logger.fine("Transferring " + transfer.getSource() + "->" +
                      transfer.getDest() + " size " +
                      transfer.getSize() + " bytes.");
              if (destf.push(transfer) != transfer.getSize()) {
                  throw new IOException("Invalid transfer size");
              }
          } catch (RemoteException e) {
              Throwable t = e;
              Throwable cause = t.getCause();
              while (cause != null) {
                  t = cause;
                  cause = t.getCause();
              }
  
              logger.log(Level.SEVERE, "CmdService: Pushing - " +
                      "exception writing file " + destfile, t);
              return false;
          } catch (IOException e) {
              logger.log(Level.SEVERE, "CmdService: Pushing - " +
                      "exception reading file " + srcfile, e);
              return false;
          }
          return true;
      }
  
      /**
       * Gets a remote file to the Faban master.
       * @param srcmachine The source machine
       * @param srcfile The source file name
       * @param destfile The destination file name, always full path
       * @return true if successful, false otherwise
       */
      public synchronized boolean get(String srcmachine, String srcfile,
              String destfile) {
          int sidx = machinesList.indexOf(srcmachine);
          if (sidx == -1) {
              throw new FabanHostUnknownException(
                      "Host " + srcmachine + " not found!");
          }
          try {
              String src = InetAddress.getLocalHost().getHostName();
              String dest = hostRoles.getHostByAlias(srcmachine);
              if (dest.equals(src)) {
                  if (srcfile.equals(destfile)) {
                      return true;
                  } else {
                      return FileHelper.copyFile(srcfile, destfile, true);
                  }
              }
          } catch (UnknownHostException e) {
              logger.log(Level.SEVERE, "CmdService: Cannot determine own" +
                      "host name", e);
              return false;
          }
  
          FileAgent srcf = filep.get(sidx);
          try {
              FileTransfer transfer = srcf.get(srcfile, destfile);
              if (transfer.getSize() != transfer.getTransferSize()) {
                  throw new IOException("Received " + transfer.getSource() +
                          "->" + transfer.getDest() + ", " +
                          transfer.getTransferSize() + " out of " +
                          transfer.getSize() + " bytes");
              }
          } catch (RemoteException e) {
              Throwable t = e;
              Throwable cause = t.getCause();
              while (cause != null) {
                  t = cause;
                  cause = t.getCause();
              }
  
              logger.log(Level.SEVERE, "CmdService: Getting - " +
                      "exception reading file " + srcfile, t);
              return false;
          } catch (IOException e) {
              logger.log(Level.SEVERE, "CmdService: Getting - " +
                      "exception reading " + srcfile, e);
              return false;
          }
          return true;
      }
  
      /**
       * Copy a file from one remote machine to another
       * This method essentially does the work of 'rcp'
       * using the FileAgents on the machines.
       * @param srcmachine - Name of source machine
       * @param destmachine - Name of destination machine
       * @param srcfile - Name of source file
       * @param destfile - Name of destination file
       * @param append to dest file or overwrite it
       * @return true/false if copy was successful/failed
       * @deprecated
       */
      @Deprecated public synchronized boolean copy(String srcmachine, String destmachine,
              String srcfile, String destfile,
              boolean append) {
  
          FileAgent srcf, destf = null;
          FileService srcfilep = null, destfilep = null;
          int sidx = machinesList.indexOf(srcmachine);
          int didx = machinesList.indexOf(destmachine);
          byte[] buf;
          if (sidx == didx && srcfile.equals(destfile)) {
              return (true);
          }
  
          if (srcfile.equals(destfile)) {
              try {
                  String dest = cmdp.get(didx).getHostName();
                  String src = cmdp.get(sidx).getHostName();
                  if (dest == src) {
                      return true;
                  }
              } catch (Exception e) {
                  logger.severe("CmdService: Copying - CmdAgent getHostName exception");
                  logger.log(Level.FINE, "Exception", e);
              }
          }
          logger.fine("CmdService: Copying " + srcfile + " from " + srcmachine + " to " + destfile + " in " + destmachine);
  
          srcf = filep.get(sidx);
          destf = filep.get(didx);
          try {
              srcfilep = srcf.open(srcfile, FileAgent.READ);
              if (append) {
                  destfilep = destf.open(destfile, FileAgent.APPEND);
              } else {
                  destfilep = destf.open(destfile, FileAgent.WRITE);
              }
  
              // Read from src and write to dest.
              buf = srcfilep.read();
              destfilep.write(buf);
  
              srcfilep.close();
              destfilep.close();
          } catch (Exception ie) {
              logger.log(Level.WARNING, "CmdService: Could not copy " +
                      srcmachine + ":" + srcfile + " to " + destmachine + ":" +
                      destfile, ie);
              return (false);
          }
          return true;
      }
  
      /**
       * Obtains the temporary dircteroy for the given machine.
       *
       * @param machine The machine name
       * @return The temporary directory to use on the machine
       */
      public String getTmpDir(String machine) {
          try {
              return findCmdAgent(machine).getTmpDir();
          } catch (RemoteException e) {
              throw new RuntimeException(e);
          }
      }
  
      private CmdAgent findCmdAgent(String machine) {
          if (machine == null || machine.length() == 0) {
              throw new IllegalArgumentException(
                      "Machine cannot be null or zero length");
          }
          int index = machinesList.indexOf(machine);
          if (index == -1) {
              throw new FabanHostUnknownException(
                      "Host " + machine + " not found!");
          }
          return cmdp.get(index);
      }
  
      private FileAgent findFileAgent(String machine) {
          if (machine == null || machine.length() == 0) {
              throw new IllegalArgumentException(
                      "Machine cannot be null or zero length");
          }
          int index = machinesList.indexOf(machine);
          if (index == -1) {
              throw new FabanHostUnknownException(
                      "Host " + machine + " not found!");
          }
          return filep.get(index);
      }
  
      /**
       * Deletes the file from the machine.
       *
       * @param srcmachine The machine name
       * @param srcfile The file name
       * @return true if the file has been deleted, false otherwise
       */
      public synchronized boolean delete(String srcmachine, String srcfile) {
          try {
              return findFileAgent(srcmachine).removeFile(srcfile);
          } catch (Exception ie) {
              logger.severe("CmdService: Could not delete " + srcmachine +
                      ":" + srcfile);
              logger.log(Level.FINE, "Exception", ie);
              return false;
          }
      }
  
      /**
       * Deletes the file from the machine based on the filter provided.
       *
       * @param srcmachine The machine name
       * @param dir The directory name
       * @param filter The file filter to use
       * @return true if akk files selected by the filter has been removed
       */
      public synchronized boolean delete(String srcmachine, String dir,
              com.sun.faban.harness.FileFilter filter) {
          try {
              return findFileAgent(srcmachine).removeFiles(dir, filter);
          } catch (Exception ie) {
              logger.severe("CmdService: Could not delete files on " +
                      srcmachine + ":" + dir);
              logger.log(Level.FINE, "Exception", ie);
              return false;
          }
      }
      /**
       * Truncates the file from the machine.
       *
       * @param srcmachine The machine name
       * @param srcfile The file name
       * @return true if the file has been deleted, false otherwise
       */
      public synchronized boolean truncate(String srcmachine, String srcfile) {
          try {
              return findFileAgent(srcmachine).truncateFile(srcfile);
          } catch (Exception ie) {
              logger.severe("CmdService: Could not truncate " + srcmachine +
                      ":" + srcfile);
              logger.log(Level.FINE, "Exception", ie);
              return false;
          }
      }
  
  
      /**
       * Copy a file from one remote machine to a stream on the master.
       * This method essentially does the work of 'rcp'
       * using the FileAgents on the machines.
       * @param srcmachine - Name of source machine
       * @param srcfile - Name of source file
       * @param stream The stream to copy the content to
       * @return true/false if copy was successful/failed
       */
      public synchronized boolean copyToStream(String srcmachine, String srcfile,
              OutputStream stream) {
          FileService srcfilep = null;
          byte[] buf = null;
  
          FileAgent srcf = findFileAgent(srcmachine);
          try {
              srcfilep = srcf.open(srcfile, FileAgent.READ);
  
              // Now loop, reading from src and writing to dest
              while (true) {
                  buf = srcfilep.readBytes(1000000);
                  //	logger.info("           Read " + buf);
                  //	logger.info(buf);
                  //		logger.info(buf.length);
                  stream.write(buf);
                  if (buf.length < 1000000) {
                      break;
                  }
              }
  
              srcfilep.close();
          } catch (Exception ie) {
              logger.log(Level.WARNING, "CmdService: Could not copy " +
                      srcmachine + ":" + srcfile, ie);
              return (false);
          }
          return (true);
      }
  
      /**
       * Set the Log level for Agents.
       * @param name Logger name
       * @param level Log level
       */
      public void setLogLevel(String name, Level level) {
          int i = 0;
          try {
              for (i = 0; i < cmdp.size(); i++) {
                  cmdp.get(i).setLogLevel(name, level);
              }
          } catch (Exception e) {
              logger.severe(" setLogLevel Failed for CmdAgent@" + machinesList.get(i));
              logger.log(Level.FINE, "Exception", e);
          }
      }
  
      /**
       * Obtains the registry.
       * @return The registry
       */
      public Registry getRegistry() {
          return registry;
      }
  
      /**
       * Checks whether the given remote file exists.
       * @param hostName The host name to check.
       * @param fileName The file name to test.
       * @return true if exists, false otherwise.
       */
      public boolean doesFileExist(String hostName, String fileName) {
          try {
              return findFileAgent(hostName).doesFileExist(fileName);
          } catch (Exception ie) {
              logger.log(Level.SEVERE, "CmdService: Could not check " +
                      hostName + ":" + fileName, ie);
              return false;
          }
      }
  
      /**
       * Checks whether the given remote file exists and is a normal file.
       * @param hostName The host name to check.
       * @param fileName The file name to test.
       * @return true if file is a normal file, false otherwise.
       */
      public boolean isFile(String hostName, String fileName) {
          try {
              return findFileAgent(hostName).isFile(fileName);
          } catch (Exception ie) {
              logger.log(Level.SEVERE, "CmdService: Could not check " +
                      hostName + ":" + fileName, ie);
              return false;
          }
      }
  
      /**
       * Checks whether the given remote file exists and is a directory.
       * @param hostName The host name to check.
       * @param fileName The file name to test.
       * @return true if file is a directory, false otherwise.
       */
      public boolean isDirectory(String hostName, String fileName) {
          try {
              return findFileAgent(hostName).isDirectory(fileName);
          } catch (Exception ie) {
              logger.log(Level.SEVERE, "CmdService: Could not check " +
                      hostName + ":" + fileName, ie);
              return false;
          }
      }
  }
  