content: /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.solr.handler;
  
  import org.apache.commons.httpclient.*;
  import org.apache.commons.httpclient.auth.AuthScope;
  import org.apache.commons.httpclient.methods.PostMethod;
  import org.apache.commons.io.IOUtils;
  import org.apache.lucene.index.IndexCommit;
  import org.apache.solr.common.SolrException;
  import org.apache.solr.common.util.FastInputStream;
  import org.apache.solr.common.util.JavaBinCodec;
  import org.apache.solr.common.util.NamedList;
  import org.apache.solr.common.util.FileUtils;
  import org.apache.solr.core.SolrCore;
  import static org.apache.solr.handler.ReplicationHandler.*;
  import org.apache.solr.search.SolrIndexSearcher;
  import org.apache.solr.update.CommitUpdateCommand;
  import org.apache.solr.update.DirectUpdateHandler2;
  import org.apache.solr.util.RefCounted;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import java.io.*;
  import java.nio.ByteBuffer;
  import java.nio.channels.FileChannel;
  import java.text.SimpleDateFormat;
  import java.util.*;
  import java.util.concurrent.*;
  import java.util.concurrent.atomic.AtomicBoolean;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  import java.util.zip.Adler32;
  import java.util.zip.Checksum;
  import java.util.zip.GZIPInputStream;
  import java.util.zip.InflaterInputStream;
  
  /**
   * <p/> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
   * master. </p>
   *
   * @version $Id: SnapPuller.java 1065312 2011-01-30 16:08:25Z rmuir $
   * @since solr 1.4
   */
  
  public class SnapPuller{}
  //
  //public class SnapPuller {
  //  private static final Logger LOG = LoggerFactory.getLogger(SnapPuller.class.getName());
  //
  //  private final String masterUrl;
  //
  //  private final ReplicationHandler replicationHandler;
  //
  //  private final Integer pollInterval;
  //
  //  private String pollIntervalStr;
  //
  //  private ScheduledExecutorService executorService;
  //
  //  private volatile long executorStartTime;
  //
  //  private volatile long replicationStartTime;
  //
  //  private final SolrCore solrCore;
  //
  //  private volatile List<Map<String, Object>> filesToDownload;
  //
  //  private volatile List<Map<String, Object>> confFilesToDownload;
  //
  //  private volatile List<Map<String, Object>> filesDownloaded;
  //
  //  private volatile List<Map<String, Object>> confFilesDownloaded;
  //
  //  private volatile Map<String, Object> currentFile;
  //
  //  private volatile FileFetcher fileFetcher;
  //
  //  private volatile ExecutorService fsyncService;
  //
  //  private volatile boolean stop = false;
  //
  //  private boolean useInternal = false;
  //
  //  private boolean useExternal = false;
  //
  //  /**
  //   * Disable the timer task for polling
  //   */
  //  private AtomicBoolean pollDisabled = new AtomicBoolean(false);
  //
  //  // HttpClient shared by all cores (used if timeout is not specified for a core)
  //  private static HttpClient client;
  //  // HttpClient for this instance if connectionTimeout or readTimeout has been specified
  //  private final HttpClient myHttpClient;
  //
  //  private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout) {
  //    if (connTimeout == null && readTimeout == null && client != null)  return client;
  //    MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
  //    // Keeping a very high number so that if you have a large number of cores
  //    // no requests are kept waiting for an idle connection.
  //    mgr.getParams().setDefaultMaxConnectionsPerHost(10000);
  //    mgr.getParams().setMaxTotalConnections(10000);
  //    mgr.getParams().setSoTimeout(readTimeout == null ? 20000 : Integer.parseInt(readTimeout)); //20 secs
  //    mgr.getParams().setConnectionTimeout(connTimeout == null ? 5000 : Integer.parseInt(connTimeout)); //5 secs
  //    HttpClient httpClient = new HttpClient(mgr);
  //    if (client == null && connTimeout == null && readTimeout == null) client = httpClient;
  //    return httpClient;
  //  }
  //
  //  public SnapPuller(NamedList initArgs, ReplicationHandler handler, SolrCore sc) {
  //    solrCore = sc;
  //    masterUrl = (String) initArgs.get(MASTER_URL);
  //    if (masterUrl == null)
  //      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //              "'masterUrl' is required for a slave");
  //    this.replicationHandler = handler;
  //    pollIntervalStr = (String) initArgs.get(POLL_INTERVAL);
  //    pollInterval = readInterval(pollIntervalStr);
  //    String compress = (String) initArgs.get(COMPRESSION);
  //    useInternal = INTERNAL.equals(compress);
  //    useExternal = EXTERNAL.equals(compress);
  //    String connTimeout = (String) initArgs.get(HTTP_CONN_TIMEOUT);
  //    String readTimeout = (String) initArgs.get(HTTP_READ_TIMEOUT);
  //    String httpBasicAuthUser = (String) initArgs.get(HTTP_BASIC_AUTH_USER);
  //    String httpBasicAuthPassword = (String) initArgs.get(HTTP_BASIC_AUTH_PASSWORD);
  //    myHttpClient = createHttpClient(connTimeout, readTimeout);
  //    if (httpBasicAuthUser != null && httpBasicAuthPassword != null) {
  //      myHttpClient.getState().setCredentials(AuthScope.ANY,
  //              new UsernamePasswordCredentials(httpBasicAuthUser, httpBasicAuthPassword));
  //    }
  //    if (pollInterval != null && pollInterval > 0) {
  //      startExecutorService();
  //    } else {
  //      LOG.info(" No value set for 'pollInterval'. Timer Task not started.");
  //    }
  //  }
  //
  //  private void startExecutorService() {
  //    Runnable task = new Runnable() {
  //      public void run() {
  //        if (pollDisabled.get()) {
  //          LOG.info("Poll disabled");
  //          return;
  //        }
  //        try {
  //          executorStartTime = System.currentTimeMillis();
  //          replicationHandler.doFetch(null);
  //        } catch (Exception e) {
  //          LOG.error("Exception in fetching index", e);
  //        }
  //      }
  //    };
  //    executorService = Executors.newSingleThreadScheduledExecutor();
  //    long initialDelay = pollInterval - (System.currentTimeMillis() % pollInterval);
  //    executorService.scheduleAtFixedRate(task, initialDelay, pollInterval, TimeUnit.MILLISECONDS);
  //    LOG.info("Poll Scheduled at an interval of " + pollInterval + "ms");
  //  }
  //
  //  /**
  //   * Gets the latest commit version and generation from the master
  //   */
  //  @SuppressWarnings("unchecked")
  //  NamedList getLatestVersion() throws IOException {
  //    PostMethod post = new PostMethod(masterUrl);
  //    post.addParameter(COMMAND, CMD_INDEX_VERSION);
  //    post.addParameter("wt", "javabin");
  //    return getNamedListResponse(post);
  //  }
  //
  //  NamedList getCommandResponse(NamedList<String> commands) throws IOException {
  //    PostMethod post = new PostMethod(masterUrl);
  //    for (Map.Entry<String, String> c : commands) {
  //      post.addParameter(c.getKey(),c.getValue());
  //    }
  //    post.addParameter("wt", "javabin");
  //    return getNamedListResponse(post);
  //  }
  //
  //  private NamedList getNamedListResponse(PostMethod method) throws IOException {
  //    try {
  //      int status = myHttpClient.executeMethod(method);
  //      if (status != HttpStatus.SC_OK) {
  //        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
  //                "Request failed for the url " + method);
  //      }
  //      return (NamedList) new JavaBinCodec().unmarshal(method.getResponseBodyAsStream());
  //    } finally {
  //      try {
  //        method.releaseConnection();
  //      } catch (Exception e) {
  //      }
  //    }
  //  }
  //
  //  /**
  //   * Fetches the list of files in a given index commit point
  //   */
  //  void fetchFileList(long version) throws IOException {
  //    PostMethod post = new PostMethod(masterUrl);
  //    post.addParameter(COMMAND, CMD_GET_FILE_LIST);
  //    post.addParameter(CMD_INDEX_VERSION, String.valueOf(version));
  //    post.addParameter("wt", "javabin");
  //    NamedList nl = getNamedListResponse(post);
  //    List<Map<String, Object>> f = (List<Map<String, Object>>) nl.get(CMD_GET_FILE_LIST);
  //    if (f != null)
  //      filesToDownload = Collections.synchronizedList(f);
  //    else {
  //      filesToDownload = Collections.emptyList();
  //      LOG.error("No files to download for indexversion: "+ version);
  //    }
  //
  //    f = (List<Map<String, Object>>) nl.get(CONF_FILES);
  //    if (f != null)
  //      confFilesToDownload = Collections.synchronizedList(f);
  //  }
  //
  //  /**
  //   * This command downloads all the necessary files from master to install a index commit point. Only changed files are
  //   * downloaded. It also downloads the conf files (if they are modified).
  //   *
  //   * @param core the SolrCore
  //   * @return true on success, false if slave is already in sync
  //   * @throws IOException if an exception occurs
  //   */
  //  @SuppressWarnings("unchecked")
  //  boolean successfulInstall = false;
  //
  //  boolean fetchLatestIndex(SolrCore core) throws IOException {
  //    replicationStartTime = System.currentTimeMillis();
  //    try {
  //      //get the current 'replicateable' index version in the master
  //      NamedList response = null;
  //      try {
  //        response = getLatestVersion();
  //      } catch (Exception e) {
  //        LOG.error("Master at: " + masterUrl + " is not available. Index fetch failed. Exception: " + e.getMessage());
  //        return false;
  //      }
  //      long latestVersion = (Long) response.get(CMD_INDEX_VERSION);
  //      long latestGeneration = (Long) response.get(GENERATION);
  //      if (latestVersion == 0L) {
  //        //there is nothing to be replicated
  //        return false;
  //      }
  //      IndexCommit commit;
  //      RefCounted<SolrIndexSearcher> searcherRefCounted = null;
  //      try {
  //        searcherRefCounted = core.getNewestSearcher(false);
  //        commit = searcherRefCounted.get().getReader().getIndexCommit();
  //      } finally {
  //        if (searcherRefCounted != null)
  //          searcherRefCounted.decref();
  //      }
  //      if (commit.getVersion() == latestVersion && commit.getGeneration() == latestGeneration) {
  //        //master and slave are alsready in sync just return
  //        LOG.info("Slave in sync with master.");
  //        return false;
  //      }
  //      LOG.info("Master's version: " + latestVersion + ", generation: " + latestGeneration);
  //      LOG.info("Slave's version: " + commit.getVersion() + ", generation: " + commit.getGeneration());
  //      LOG.info("Starting replication process");
  //      // get the list of files first
  //      fetchFileList(latestVersion);
  //      // this can happen if the commit point is deleted before we fetch the file list.
  //      if(filesToDownload.isEmpty()) return false;
  //      LOG.info("Number of files in latest index in master: " + filesToDownload.size());
  //
  //      // Create the sync service
  //      fsyncService = Executors.newSingleThreadExecutor();
  //      // use a synchronized list because the list is read by other threads (to show details)
  //      filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
  //      // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
  //      // then a new index direcory to be created and all the files need to be copied
  //      boolean isFullCopyNeeded = commit.getGeneration() >= latestGeneration;
  //      File tmpIndexDir = createTempindexDir(core);
  //      if (isIndexStale())
  //        isFullCopyNeeded = true;
  //      successfulInstall = false;
  //      boolean deleteTmpIdxDir = true;
  //      File indexDir = null ;
  //      try {
  //        indexDir = new File(core.getIndexDir());
  //        downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestVersion);
  //        LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime) / 1000) + " secs");
  //        Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
  //        if (!modifiedConfFiles.isEmpty()) {
  //          downloadConfFiles(confFilesToDownload, latestVersion);
  //          if (isFullCopyNeeded) {
  //            successfulInstall = modifyIndexProps(tmpIndexDir.getName());
  //            deleteTmpIdxDir =  false;
  //          } else {
  //            successfulInstall = copyIndexFiles(tmpIndexDir, indexDir);
  //          }
  //          if (successfulInstall) {
  //            LOG.info("Configuration files are modified, core will be reloaded");
  //            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);//write to a file time of replication and conf files.
  //            reloadCore();
  //          }
  //        } else {
  //          terminateAndWaitFsyncService();
  //          if (isFullCopyNeeded) {
  //            successfulInstall = modifyIndexProps(tmpIndexDir.getName());
  //            deleteTmpIdxDir =  false;
  //          } else {
  //            successfulInstall = copyIndexFiles(tmpIndexDir, indexDir);
  //          }
  //          if (successfulInstall) {
  //            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
  //            doCommit();
  //          }
  //        }
  //        replicationStartTime = 0;
  //        return successfulInstall;
  //      } catch (ReplicationHandlerException e) {
  //        LOG.error("User aborted Replication");
  //      } catch (SolrException e) {
  //        throw e;
  //      } catch (Exception e) {
  //        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Index fetch failed : ", e);
  //      } finally {
  //        if (deleteTmpIdxDir) delTree(tmpIndexDir);
  //        else delTree(indexDir);
  //      }
  //      return successfulInstall;
  //    } finally {
  //      if (!successfulInstall) {
  //        logReplicationTimeAndConfFiles(null, successfulInstall);
  //      }
  //      filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = null;
  //      replicationStartTime = 0;
  //      fileFetcher = null;
  //      if (fsyncService != null && !fsyncService.isShutdown()) fsyncService.shutdownNow();
  //      fsyncService = null;
  //      stop = false;
  //      fsyncException = null;
  //    }
  //  }
  //
  //  private volatile Exception fsyncException;
  //
  //  /**
  //   * terminate the fsync service and wait for all the tasks to complete. If it is already terminated
  //   *
  //   * @throws Exception
  //   */
  //  private void terminateAndWaitFsyncService() throws Exception {
  //    if (fsyncService.isTerminated()) return;
  //    fsyncService.shutdown();
  //     // give a long wait say 1 hr
  //    fsyncService.awaitTermination(3600, TimeUnit.SECONDS);
  //    // if any fsync failed, throw that exception back
  //    Exception fsyncExceptionCopy = fsyncException;
  //    if (fsyncExceptionCopy != null) throw fsyncExceptionCopy;
  //  }
  //
  //  /**
  //   * Helper method to record the last replication's details so that we can show them on the statistics page across
  //   * restarts.
  //   */
  //  private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>> modifiedConfFiles, boolean successfulInstall) {
  //    FileOutputStream outFile = null;
  //    List<String> confFiles = new ArrayList<String>();
  //    if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty())
  //      for (Map<String, Object> map1 : modifiedConfFiles)
  //        confFiles.add((String) map1.get(NAME));
  //
  //    Properties props = replicationHandler.loadReplicationProperties();
  //    long replicationTime = System.currentTimeMillis();
  //    long replicationTimeTaken = (replicationTime - getReplicationStartTime()) / 1000;
  //    try {
  //      int indexCount = 1, confFilesCount = 1;
  //      if (props.containsKey(TIMES_INDEX_REPLICATED)) {
  //        indexCount = Integer.valueOf(props.getProperty(TIMES_INDEX_REPLICATED)) + 1;
  //      }
  //      StringBuffer sb = readToStringBuffer(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST));
  //      props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString());
  //      props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime));
  //      props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken));
  //      props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount));
  //      if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) {
  //        props.setProperty(CONF_FILES_REPLICATED, confFiles.toString());
  //        props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime));
  //        if (props.containsKey(TIMES_CONFIG_REPLICATED)) {
  //          confFilesCount = Integer.valueOf(props.getProperty(TIMES_CONFIG_REPLICATED)) + 1;
  //        }
  //        props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount));
  //      }
  //
  //      props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded(this)));
  //      if (!successfulInstall) {
  //        int numFailures = 1;
  //        if (props.containsKey(TIMES_FAILED)) {
  //          numFailures = Integer.valueOf(props.getProperty(TIMES_FAILED)) + 1;
  //        }
  //        props.setProperty(TIMES_FAILED, String.valueOf(numFailures));
  //        props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime));
  //        sb = readToStringBuffer(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
  //        props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
  //      }
  //      File f = new File(solrCore.getDataDir(), REPLICATION_PROPERTIES);
  //      outFile = new FileOutputStream(f);
  //      props.store(outFile, "Replication details");
  //      outFile.close();
  //    } catch (Exception e) {
  //      LOG.warn("Exception while updating statistics", e);
  //    }
  //    finally {
  //      IOUtils.closeQuietly(outFile);
  //    }
  //  }
  //
  //  static long getTotalBytesDownloaded(SnapPuller snappuller) {
  //    long bytesDownloaded = 0;
  //    //get size from list of files to download
  //    for (Map<String, Object> file : snappuller.getFilesDownloaded()) {
  //      bytesDownloaded += (Long) file.get(SIZE);
  //    }
  //
  //    //get size from list of conf files to download
  //    for (Map<String, Object> file : snappuller.getConfFilesDownloaded()) {
  //      bytesDownloaded += (Long) file.get(SIZE);
  //    }
  //
  //    //get size from current file being downloaded
  //    Map<String, Object> currentFile = snappuller.getCurrentFile();
  //    if (currentFile != null) {
  //      if (currentFile.containsKey("bytesDownloaded")) {
  //        bytesDownloaded += (Long) currentFile.get("bytesDownloaded");
  //      }
  //    }
  //    return bytesDownloaded;
  //  }
  //
  //  private StringBuffer readToStringBuffer(long replicationTime, String str) {
  //    StringBuffer sb = new StringBuffer();
  //    List<String> l = new ArrayList<String>();
  //    if (str != null && str.length() != 0) {
  //      String[] ss = str.split(",");
  //      for (int i = 0; i < ss.length; i++) {
  //        l.add(ss[i]);
  //      }
  //    }
  //    sb.append(replicationTime);
  //    if (!l.isEmpty()) {
  //      for (int i = 0; i < l.size() || i < 9; i++) {
  //        if (i == l.size() || i == 9) break;
  //        String s = l.get(i);
  //        sb.append(",").append(s);
  //      }
  //    }
  //    return sb;
  //  }
  //
  //  private void doCommit() throws IOException {
  //    CommitUpdateCommand cmd = new CommitUpdateCommand(false);
  //    cmd.waitFlush = true;
  //    cmd.waitSearcher = true;
  //    solrCore.getUpdateHandler().commit(cmd);
  //    if (solrCore.getUpdateHandler() instanceof DirectUpdateHandler2) {
  //      LOG.info("Force open index writer to make sure older index files get deleted");
  //      DirectUpdateHandler2 handler = (DirectUpdateHandler2) solrCore.getUpdateHandler();
  //      handler.forceOpenWriter();
  //      replicationHandler.refreshCommitpoint();
  //    } else  {
  //      LOG.warn("The update handler is not an instance or sub-class of DirectUpdateHandler2. " +
  //              "ReplicationHandler may not be able to cleanup un-used index files.");
  //    }
  //  }
  //
  //
  //  /**
  //   * All the files are copied to a temp dir first
  //   */
  //  private File createTempindexDir(SolrCore core) {
  //    String tmpIdxDirName = "index." + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.US).format(new Date());
  //    File tmpIdxDir = new File(core.getDataDir(), tmpIdxDirName);
  //    tmpIdxDir.mkdirs();
  //    return tmpIdxDir;
  //  }
  //
  //  private void reloadCore() {
  //    new Thread() {
  //      @Override
  //      public void run() {
  //        try {
  //          solrCore.getCoreDescriptor().getCoreContainer().reload(solrCore.getName());
  //        } catch (Exception e) {
  //          LOG.error("Could not restart core ", e);
  //        }
  //      }
  //    }.start();
  //  }
  //
  //  private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestVersion) throws Exception {
  //    LOG.info("Starting download of configuration files from master: " + confFilesToDownload);
  //    confFilesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
  //    File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date()));
  //    try {
  //      boolean status = tmpconfDir.mkdirs();
  //      if (!status) {
  //        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                "Failed to create temporary config folder: " + tmpconfDir.getName());
  //      }
  //      for (Map<String, Object> file : confFilesToDownload) {
  //        String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS));
  //        fileFetcher = new FileFetcher(tmpconfDir, file, saveAs, true, latestVersion);
  //        currentFile = file;
  //        fileFetcher.fetchFile();
  //        confFilesDownloaded.add(new HashMap<String, Object>(file));
  //      }
  //      // this is called before copying the files to the original conf dir
  //      // so that if there is an exception avoid corrupting the original files.
  //      terminateAndWaitFsyncService();
  //      copyTmpConfFiles2Conf(tmpconfDir);
  //    } finally {
  //      delTree(tmpconfDir);
  //    }
  //  }
  //
  //  /**
  //   * Download the index files. If a new index is needed, download all the files.
  //   *
  //   * @param downloadCompleteIndex is it a fresh index copy
  //   * @param tmpIdxDir               the directory to which files need to be downloadeed to
  //   * @param latestVersion         the version number
  //   */
  //  private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestVersion) throws Exception {
  //    for (Map<String, Object> file : filesToDownload) {
  //      File localIndexFile = new File(solrCore.getIndexDir(), (String) file.get(NAME));
  //      if (!localIndexFile.exists() || downloadCompleteIndex) {
  //        fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestVersion);
  //        currentFile = file;
  //        fileFetcher.fetchFile();
  //        filesDownloaded.add(new HashMap<String, Object>(file));
  //      } else {
  //        LOG.info("Skipping download for " + localIndexFile);
  //      }
  //    }
  //  }
  //
  //  /**
  //   * All the files which are common between master and slave must have same timestamp and size else we assume they are
  //   * not compatible (stale).
  //   *
  //   * @return true if the index stale and we need to download a fresh copy, false otherwise.
  //   */
  //  private boolean isIndexStale() {
  //    for (Map<String, Object> file : filesToDownload) {
  //      File localIndexFile = new File(solrCore.getIndexDir(), (String) file
  //              .get(NAME));
  //      if (localIndexFile.exists()
  //              && localIndexFile.length() != (Long) file.get(SIZE)) {
  //        // file exists and size is different, therefore we must assume
  //        // corrupted index
  //        return true;
  //      }
  //    }
  //    return false;
  //  }
  //
  //  /**
  //   * Copy a file by the File#renameTo() method. If it fails, it is considered a failure
  //   * <p/>
  //   */
  //  private boolean copyAFile(File tmpIdxDir, File indexDir, String fname, List<String> copiedfiles) {
  //    File indexFileInTmpDir = new File(tmpIdxDir, fname);
  //    File indexFileInIndex = new File(indexDir, fname);
  //    boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
  //    if(!success){
  //      try {
  //        LOG.error("Unable to move index file from: " + indexFileInTmpDir
  //              + " to: " + indexFileInIndex + "Trying to do a copy");
  //        FileUtils.copyFile(indexFileInTmpDir,indexFileInIndex);
  //        success = true;
  //      } catch (IOException e) {
  //        LOG.error("Unable to copy index file from: " + indexFileInTmpDir
  //              + " to: " + indexFileInIndex , e);
  //      }
  //    }
  //    if (!success) {
  //      for (String f : copiedfiles) {
  //        File indexFile = new File(indexDir, f);
  //        if (indexFile.exists())
  //          indexFile.delete();
  //      }
  //      delTree(tmpIdxDir);
  //      return false;
  //    }
  //    return true;
  //  }
  //
  //  /**
  //   * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last.
  //   */
  //  private boolean copyIndexFiles(File tmpIdxDir, File indexDir) throws IOException {
  //    String segmentsFile = null;
  //    List<String> copiedfiles = new ArrayList<String>();
  //    for (Map<String, Object> f : filesDownloaded) {
  //      String fname = (String) f.get(NAME);
  //      // the segments file must be copied last
  //      // or else if there is a failure in between the
  //      // index will be corrupted
  //      if (fname.startsWith("segments_")) {
  //        //The segments file must be copied in the end
  //        //Otherwise , if the copy fails index ends up corrupted
  //        segmentsFile = fname;
  //        continue;
  //      }
  //      if (!copyAFile(tmpIdxDir, indexDir, fname, copiedfiles)) return false;
  //      copiedfiles.add(fname);
  //    }
  //    //copy the segments file last
  //    if (segmentsFile != null) {
  //      if (!copyAFile(tmpIdxDir, indexDir, segmentsFile, copiedfiles)) return false;
  //    }
  //    return true;
  //  }
  //
  //  /**
  //   * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained
  //   */
  //  private void copyTmpConfFiles2Conf(File tmpconfDir) throws IOException {
  //    File confDir = new File(solrCore.getResourceLoader().getConfigDir());
  //    for (File file : tmpconfDir.listFiles()) {
  //      File oldFile = new File(confDir, file.getName());
  //      if (oldFile.exists()) {
  //        File backupFile = new File(confDir, oldFile.getName() + "." + getDateAsStr(new Date(oldFile.lastModified())));
  //        boolean status = oldFile.renameTo(backupFile);
  //        if (!status) {
  //          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                  "Unable to rename: " + oldFile + " to: " + backupFile);
  //        }
  //      }
  //      boolean status = file.renameTo(oldFile);
  //      if (status) {
  //      } else {
  //        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                "Unable to rename: " + file + " to: " + oldFile);
  //      }
  //    }
  //  }
  //
  //  private String getDateAsStr(Date d) {
  //    return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.US).format(d);
  //  }
  //
  //  /**
  //   * If the index is stale by any chance, load index from a different dir in the data dir.
  //   */
  //  private boolean modifyIndexProps(String tmpIdxDirName) {
  //    LOG.info("New index installed. Updating index properties...");
  //    File idxprops = new File(solrCore.getDataDir() + "index.properties");
  //    Properties p = new Properties();
  //    if (idxprops.exists()) {
  //      InputStream is = null;
  //      try {
  //        is = new FileInputStream(idxprops);
  //        p.load(is);
  //      } catch (Exception e) {
  //        LOG.error("Unable to load index.properties");
  //      } finally {
  //        IOUtils.closeQuietly(is);
  //      }
  //    }
  //    p.put("index", tmpIdxDirName);
  //    FileOutputStream os = null;
  //    try {
  //      os = new FileOutputStream(idxprops);
  //      p.store(os, "index properties");
  //    } catch (Exception e) {
  //      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //              "Unable to write index.properties", e);
  //    } finally {
  //      IOUtils.closeQuietly(os);
  //    }
  //      return true;
  //  }
  //
  //  private final Map<String, FileInfo> confFileInfoCache = new HashMap<String, FileInfo>();
  //
  //  /**
  //   * The local conf files are compared with the conf files in the master. If they are same (by checksum) do not copy.
  //   *
  //   * @param confFilesToDownload The list of files obtained from master
  //   *
  //   * @return a list of configuration files which have changed on the master and need to be downloaded.
  //   */
  //  private Collection<Map<String, Object>> getModifiedConfFiles(List<Map<String, Object>> confFilesToDownload) {
  //    if (confFilesToDownload == null || confFilesToDownload.isEmpty())
  //      return Collections.EMPTY_LIST;
  //    //build a map with alias/name as the key
  //    Map<String, Map<String, Object>> nameVsFile = new HashMap<String, Map<String, Object>>();
  //    NamedList names = new NamedList();
  //    for (Map<String, Object> map : confFilesToDownload) {
  //      //if alias is present that is the name the file may have in the slave
  //      String name = (String) (map.get(ALIAS) == null ? map.get(NAME) : map.get(ALIAS));
  //      nameVsFile.put(name, map);
  //      names.add(name, null);
  //    }
  //    //get the details of the local conf files with the same alias/name
  //    List<Map<String, Object>> localFilesInfo = replicationHandler.getConfFileInfoFromCache(names, confFileInfoCache);
  //    //compare their size/checksum to see if
  //    for (Map<String, Object> fileInfo : localFilesInfo) {
  //      String name = (String) fileInfo.get(NAME);
  //      Map<String, Object> m = nameVsFile.get(name);
  //      if (m == null) continue; // the file is not even present locally (so must be downloaded)
  //      if (m.get(CHECKSUM).equals(fileInfo.get(CHECKSUM))) {
  //        nameVsFile.remove(name); //checksums are same so the file need not be downloaded
  //      }
  //    }
  //    return nameVsFile.isEmpty() ? Collections.EMPTY_LIST : nameVsFile.values();
  //  }
  //
  //  /**
  //   * Delete the directory tree recursively
  //   */
  //  static boolean delTree(File dir) {
  //    if (dir == null || !dir.exists())
  //      return false;
  //    boolean isSuccess = true;
  //    File contents[] = dir.listFiles();
  //    if (contents != null) {
  //      for (File file : contents) {
  //        if (file.isDirectory()) {
  //          boolean success = delTree(file);
  //          if (!success) {
  //            LOG.warn("Unable to delete directory : " + file);
  //            isSuccess = false;
  //          }
  //        } else {
  //          boolean success = file.delete();
  //          if (!success) {
  //            LOG.warn("Unable to delete file : " + file);
  //            isSuccess = false;
  //            return false;
  //          }
  //        }
  //      }
  //    }
  //    return isSuccess && dir.delete();
  //  }
  //
  //  /**
  //   * Disable periodic polling
  //   */
  //  void disablePoll() {
  //    pollDisabled.set(true);
  //    LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled);
  //  }
  //
  //  /**
  //   * Enable periodic polling
  //   */
  //  void enablePoll() {
  //    pollDisabled.set(false);
  //    LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled);
  //  }
  //
  //  /**
  //   * Stops the ongoing pull
  //   */
  //  void abortPull() {
  //    stop = true;
  //  }
  //
  //  long getReplicationStartTime() {
  //    return replicationStartTime;
  //  }
  //
  //  List<Map<String, Object>> getConfFilesToDownload() {
  //    //make a copy first because it can be null later
  //    List<Map<String, Object>> tmp = confFilesToDownload;
  //    //create a new instance. or else iterator may fail
  //    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
  //  }
  //
  //  List<Map<String, Object>> getConfFilesDownloaded() {
  //    //make a copy first because it can be null later
  //    List<Map<String, Object>> tmp = confFilesDownloaded;
  //    // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList)
  //    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
  //  }
  //
  //  List<Map<String, Object>> getFilesToDownload() {
  //    //make a copy first because it can be null later
  //    List<Map<String, Object>> tmp = filesToDownload;
  //    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
  //  }
  //
  //  List<Map<String, Object>> getFilesDownloaded() {
  //    List<Map<String, Object>> tmp = filesDownloaded;
  //    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
  //  }
  //
  //  Map<String, Object> getCurrentFile() {
  //    Map<String, Object> tmp = currentFile;
  //    FileFetcher tmpFileFetcher = fileFetcher;
  //    if (tmp == null)
  //      return null;
  //    tmp = new HashMap<String, Object>(tmp);
  //    if (tmpFileFetcher != null)
  //      tmp.put("bytesDownloaded", tmpFileFetcher.bytesDownloaded);
  //    return tmp;
  //  }
  //
  //  boolean isPollingDisabled() {
  //    return pollDisabled.get();
  //  }
  //
  //  Long getNextScheduledExecTime() {
  //    Long nextTime = null;
  //    if (executorStartTime > 0)
  //      nextTime = executorStartTime + pollInterval;
  //    return nextTime;
  //  }
  //
  //  private static class ReplicationHandlerException extends InterruptedException {
  //    public ReplicationHandlerException(String message) {
  //      super(message);
  //    }
  //  }
  //
  //  /**
  //   * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of wt=filestream
  //   *
  //   * @see org.apache.solr.handler.ReplicationHandler.FileStream
  //   */
  //  private class FileFetcher {
  //    boolean includeChecksum = true;
  //
  //    private File copy2Dir;
  //
  //    String fileName;
  //
  //    String saveAs;
  //
  //    long size, lastmodified;
  //
  //    long bytesDownloaded = 0;
  //
  //    FileChannel fileChannel;
  //    
  //    private FileOutputStream fileOutputStream;
  //
  //    byte[] buf = new byte[1024 * 1024];
  //
  //    Checksum checksum;
  //
  //    File file;
  //
  //    int errorCount = 0;
  //
  //    private boolean isConf;
  //
  //    private PostMethod post;
  //
  //    private boolean aborted = false;
  //
  //    private Long indexVersion;
  //
  //    FileFetcher(File dir, Map<String, Object> fileDetails, String saveAs,
  //                boolean isConf, long latestVersion) throws IOException {
  //      this.copy2Dir = dir;
  //      this.fileName = (String) fileDetails.get(NAME);
  //      this.size = (Long) fileDetails.get(SIZE);
  //      this.isConf = isConf;
  //      this.saveAs = saveAs;
  //      if(fileDetails.get(LAST_MODIFIED) != null){
  //        lastmodified = (Long)fileDetails.get(LAST_MODIFIED);
  //      }
  //      indexVersion = latestVersion;
  //
  //      this.file = new File(copy2Dir, saveAs);
  //      
  //      this.fileOutputStream = new FileOutputStream(file);
  //      this.fileChannel = this.fileOutputStream.getChannel();
  //
  //      if (includeChecksum)
  //        checksum = new Adler32();
  //    }
  //
  //    /**
  //     * The main method which downloads file
  //     */
  //    void fetchFile() throws Exception {
  //      try {
  //        while (true) {
  //          final FastInputStream is = getStream();
  //          int result;
  //          try {
  //            //fetch packets one by one in a single request
  //            result = fetchPackets(is);
  //            if (result == 0 || result == NO_CONTENT) {
  //              // if the file is downloaded properly set the
  //              //  timestamp same as that in the server
  //              if (file.exists() && lastmodified > 0)
  //                file.setLastModified(lastmodified);
  //              return;
  //            }
  //            //if there is an error continue. But continue from the point where it got broken
  //          } finally {
  //            IOUtils.closeQuietly(is);
  //          }
  //        }
  //      } finally {
  //        cleanup();
  //        //if cleanup suceeds . The file is downloaded fully. do an fsync
  //        fsyncService.submit(new Runnable(){
  //          public void run() {
  //            try {
  //              FileUtils.sync(file);
  //            } catch (IOException e) {
  //              fsyncException = e;
  //            }
  //          }
  //        });
  //      }
  //    }
  //
  //    private int fetchPackets(FastInputStream fis) throws Exception {
  //      byte[] intbytes = new byte[4];
  //      byte[] longbytes = new byte[8];
  //      try {
  //        while (true) {
  //          if (stop) {
  //            stop = false;
  //            aborted = true;
  //            throw new ReplicationHandlerException("User aborted replication");
  //          }
  //          long checkSumServer = -1;
  //          fis.readFully(intbytes);
  //          //read the size of the packet
  //          int packetSize = readInt(intbytes);
  //          if (packetSize <= 0) {
  //            LOG.warn("No content recieved for file: " + currentFile);
  //            return NO_CONTENT;
  //          }
  //          if (buf.length < packetSize)
  //            buf = new byte[packetSize];
  //          if (checksum != null) {
  //            //read the checksum
  //            fis.readFully(longbytes);
  //            checkSumServer = readLong(longbytes);
  //          }
  //          //then read the packet of bytes
  //          fis.readFully(buf, 0, packetSize);
  //          //compare the checksum as sent from the master
  //          if (includeChecksum) {
  //            checksum.reset();
  //            checksum.update(buf, 0, packetSize);
  //            long checkSumClient = checksum.getValue();
  //            if (checkSumClient != checkSumServer) {
  //              LOG.error("Checksum not matched between client and server for: " + currentFile);
  //              //if checksum is wrong it is a problem return for retry
  //              return 1;
  //            }
  //          }
  //          //if everything is fine, write down the packet to the file
  //          fileChannel.write(ByteBuffer.wrap(buf, 0, packetSize));
  //          bytesDownloaded += packetSize;
  //          if (bytesDownloaded >= size)
  //            return 0;
  //          //errorcount is always set to zero after a successful packet
  //          errorCount = 0;
  //        }
  //      } catch (ReplicationHandlerException e) {
  //        throw e;
  //      } catch (Exception e) {
  //        LOG.warn("Error in fetching packets ", e);
  //        //for any failure , increment the error count
  //        errorCount++;
  //        //if it fails for the same pacaket for   MAX_RETRIES fail and come out
  //        if (errorCount > MAX_RETRIES) {
  //          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                  "Fetch failed for file:" + fileName, e);
  //        }
  //        return ERR;
  //      }
  //    }
  //
  //    /**
  //     * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be read as readFully()
  //     * other wise it fails. So read everything as bytes and then extract an integer out of it
  //     */
  //    private int readInt(byte[] b) {
  //      return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16)
  //              | ((b[2] & 0xff) << 8) | (b[3] & 0xff));
  //
  //    }
  //
  //    /**
  //     * Same as above but to read longs from a byte array
  //     */
  //    private long readLong(byte[] b) {
  //      return (((long) (b[0] & 0xff)) << 56) | (((long) (b[1] & 0xff)) << 48)
  //              | (((long) (b[2] & 0xff)) << 40) | (((long) (b[3] & 0xff)) << 32)
  //              | (((long) (b[4] & 0xff)) << 24) | ((b[5] & 0xff) << 16)
  //              | ((b[6] & 0xff) << 8) | ((b[7] & 0xff));
  //
  //    }
  //
  //    /**
  //     * cleanup everything
  //     */
  //    private void cleanup() {
  //      try {
  //        //close the FileOutputStream (which also closes the Channel)
  //        fileOutputStream.close();
  //      } catch (Exception e) {/* noop */
  //          LOG.error("Error closing the file stream: "+ this.saveAs ,e);
  //      }
  //      try {
  //        post.releaseConnection();
  //      } catch (Exception e) {
  //      }
  //      if (bytesDownloaded != size) {
  //        //if the download is not complete then
  //        //delete the file being downloaded
  //        try {
  //          file.delete();
  //        } catch (Exception e) {
  //          LOG.error("Error deleting file in cleanup" + e.getMessage());
  //        }
  //        //if the failure is due to a user abort it is returned nomally else an exception is thrown
  //        if (!aborted)
  //          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                  "Unable to download " + fileName + " completely. Downloaded "
  //                          + bytesDownloaded + "!=" + size);
  //      }
  //    }
  //
  //    /**
  //     * Open a new stream using HttpClient
  //     */
  //    FastInputStream getStream() throws IOException {
  //      post = new PostMethod(masterUrl);
  //      //the method is command=filecontent
  //      post.addParameter(COMMAND, CMD_GET_FILE);
  //      //add the version to download. This is used to reserve the download
  //      post.addParameter(CMD_INDEX_VERSION, indexVersion.toString());
  //      if (isConf) {
  //        //set cf instead of file for config file
  //        post.addParameter(CONF_FILE_SHORT, fileName);
  //      } else {
  //        post.addParameter(FILE, fileName);
  //      }
  //      if (useInternal) {
  //        post.addParameter(COMPRESSION, "true");
  //      }
  //      if (useExternal) {
  //        post.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
  //      }
  //      //use checksum
  //      if (this.includeChecksum)
  //        post.addParameter(CHECKSUM, "true");
  //      //wt=filestream this is a custom protocol
  //      post.addParameter("wt", FILE_STREAM);
  //      // This happen if there is a failure there is a retry. the offset=<sizedownloaded> ensures that
  //      // the server starts from the offset
  //      if (bytesDownloaded > 0) {
  //        post.addParameter(OFFSET, "" + bytesDownloaded);
  //      }
  //      myHttpClient.executeMethod(post);
  //      InputStream is = post.getResponseBodyAsStream();
  //      //wrap it using FastInputStream
  //      if (useInternal) {
  //        is = new InflaterInputStream(is);
  //      } else if (useExternal) {
  //        is = checkCompressed(post, is);
  //      }
  //      return new FastInputStream(is);
  //    }
  //  }
  //
  //  /*
  //   * This is copied from CommonsHttpSolrServer
  //   */
  //  private InputStream checkCompressed(HttpMethod method, InputStream respBody) throws IOException {
  //    Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
  //    if (contentEncodingHeader != null) {
  //      String contentEncoding = contentEncodingHeader.getValue();
  //      if (contentEncoding.contains("gzip")) {
  //        respBody = new GZIPInputStream(respBody);
  //      } else if (contentEncoding.contains("deflate")) {
  //        respBody = new InflaterInputStream(respBody);
  //      }
  //    } else {
  //      Header contentTypeHeader = method.getResponseHeader("Content-Type");
  //      if (contentTypeHeader != null) {
  //        String contentType = contentTypeHeader.getValue();
  //        if (contentType != null) {
  //          if (contentType.startsWith("application/x-gzip-compressed")) {
  //            respBody = new GZIPInputStream(respBody);
  //          } else if (contentType.startsWith("application/x-deflate")) {
  //            respBody = new InflaterInputStream(respBody);
  //          }
  //        }
  //      }
  //    }
  //    return respBody;
  //  }
  //
  //  static Integer readInterval(String interval) {
  //    if (interval == null)
  //      return null;
  //    int result = 0;
  //    if (interval != null) {
  //      Matcher m = INTERVAL_PATTERN.matcher(interval.trim());
  //      if (m.find()) {
  //        String hr = m.group(1);
  //        String min = m.group(2);
  //        String sec = m.group(3);
  //        result = 0;
  //        try {
  //          if (sec != null && sec.length() > 0)
  //            result += Integer.parseInt(sec);
  //          if (min != null && min.length() > 0)
  //            result += (60 * Integer.parseInt(min));
  //          if (hr != null && hr.length() > 0)
  //            result += (60 * 60 * Integer.parseInt(hr));
  //          result *= 1000;
  //        } catch (NumberFormatException e) {
  //          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                  INTERVAL_ERR_MSG);
  //        }
  //      } else {
  //        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
  //                INTERVAL_ERR_MSG);
  //      }
  //
  //    }
  //    return result;
  //  }
  //
  //  public void destroy() {
  //    if (executorService != null) executorService.shutdown();
  //  }
  //
  //  String getMasterUrl() {
  //    return masterUrl;
  //  }
  //
  //  String getPollInterval() {
  //    return pollIntervalStr;
  //  }
  //
  //  private static final int MAX_RETRIES = 5;
  //
  //  private static final int NO_CONTENT = 1;
  //
  //  private static final int ERR = 2;
  //
  //  public static final String REPLICATION_PROPERTIES = "replication.properties";
  //
  //  public static final String POLL_INTERVAL = "pollInterval";
  //
  //  public static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'";
  //
  //  private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)");
  //
  //  private static final String HTTP_CONN_TIMEOUT = "httpConnTimeout";
  //
  //  private static final String HTTP_READ_TIMEOUT = "httpReadTimeout";
  //
  //  private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser";
  //
  //  private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword";
  //
  //  static final String INDEX_REPLICATED_AT = "indexReplicatedAt";
  //
  //  static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated";
  //
  //  static final String CONF_FILES_REPLICATED = "confFilesReplicated";
  //
  //  static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt";
  //
  //  static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated";
  //
  //  static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded";
  //
  //  static final String TIMES_FAILED = "timesFailed";
  //
  //  static final String REPLICATION_FAILED_AT = "replicationFailedAt";
  //
  //  static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds";
  //
  //  static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList";
  //
  //  static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList";
  //}
  