content: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.hadoop.chukwa.validationframework.util;
  
  
  import java.io.DataInputStream;
  import java.io.EOFException;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileOutputStream;
  import java.io.FileWriter;
  import java.io.IOException;
  import java.net.URI;
  import org.apache.hadoop.chukwa.ChukwaArchiveKey;
  import org.apache.hadoop.chukwa.Chunk;
  import org.apache.hadoop.chukwa.ChunkImpl;
  import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
  import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
  import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.SequenceFile;
  import org.apache.log4j.Logger;
  
  public class DataOperations {
    static Logger log = Logger.getLogger(DataOperations.class);
  
    public static void copyFile(String fromFileName, String toFileName)
        throws IOException {
      File fromFile = new File(fromFileName);
      File toFile = new File(toFileName);
  
      FileInputStream from = null;
      FileOutputStream to = null;
      try {
        from = new FileInputStream(fromFile);
        to = new FileOutputStream(toFile);
        byte[] buffer = new byte[4096];
        int bytesRead;
  
        while ((bytesRead = from.read(buffer)) != -1)
          to.write(buffer, 0, bytesRead); // write
      } finally {
        if (from != null)
          try {
            from.close();
          } catch (IOException e) {
            ;
          }
        if (to != null)
          try {
            to.close();
          } catch (IOException e) {
            // ;
          }
      }
    }
  
    public static boolean validateMD5(String inputFile, String testFile) {
      // System.out.println("validateMD5 [" + inputFile + "] [" + testFile
      // + "]");
      String md5_1 = MD5.checksum(new File(inputFile));
      String md5_2 = MD5.checksum(new File(testFile));
      // System.out.println("MD5 [" + md5_1 + "] [" + md5_2 + "]");
      return md5_1.intern() == md5_2.intern();
    }
  
    public static boolean validateMD5(FileSystem fs, Path inputFile, Path testFile) {
      // System.out.println("validateMD5 [" + inputFile + "] [" + testFile
      // + "]");
      String md5_1 = MD5.checksum(fs, inputFile);
      String md5_2 = MD5.checksum(fs, testFile);
      // System.out.println("MD5 [" + md5_1 + "] [" + md5_2 + "]");
      return md5_1.intern() == md5_2.intern();
    }
  
    public static boolean validateChukwaRecords(FileSystem fs,
        Configuration conf, Path inputFile, Path testFile) {
      SequenceFile.Reader goldReader = null;
      SequenceFile.Reader testReader = null;
      try {
        // log.info(">>>>>>>>>>>>>> Openning records [" + inputFile.getName()
        // +"][" + testFile.getName() +"]");
        goldReader = new SequenceFile.Reader(fs, inputFile, conf);
        testReader = new SequenceFile.Reader(fs, testFile, conf);
  
        ChukwaRecordKey goldKey = new ChukwaRecordKey();
        ChukwaRecord goldRecord = new ChukwaRecord();
  
        ChukwaRecordKey testKey = new ChukwaRecordKey();
        ChukwaRecord testRecord = new ChukwaRecord();
  
        // log.info(">>>>>>>>>>>>>> Start reading");
        while (goldReader.next(goldKey, goldRecord)) {
          testReader.next(testKey, testRecord);
  
          if (goldKey.compareTo(testKey) != 0) {
            log.info(">>>>>>>>>>>>>> Not the same Key");
            log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
                + goldKey.getReduceType() + "]");
            log.info(">>>>>>>>>>>>>> Record [" + testKey.getKey() + "] ["
                + testKey.getReduceType() + "]");
            return false;
          }
  
          if (goldRecord.compareTo(testRecord) != 0) {
            log.info(">>>>>>>>>>>>>> Not the same Value");
            log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
                + goldKey.getReduceType() + "]");
            log.info(">>>>>>>>>>>>>> Record [" + testKey.getKey() + "] ["
                + testKey.getReduceType() + "]");
            log.info(">>>>>>>>>>>>>> Gold Value [" + goldRecord.toString() + "]");
            log.info(">>>>>>>>>>>>>> Test value [" + testRecord.toString() + "]");
  
            return false;
          }
        }
        // log.info(">>>>>>>>>>>>>> Same File");
        return true;
      } catch (IOException e) {
        e.printStackTrace();
        return false;
      } finally {
        try {
          goldReader.close();
          testReader.close();
        } catch (IOException e) {
        }
  
      }
    }
  
    public static void extractRawLogFromdataSink(String directory, String fileName)
        throws Exception {
      ChukwaConfiguration conf = new ChukwaConfiguration();
      String fsName = conf.get("writer.hdfs.filesystem");
      FileSystem fs = FileSystem.get(new URI(fsName), conf);
  
      SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(directory
          + fileName + ".done"), conf);
  
      File outputFile = new File(directory + fileName + ".raw");
  
      ChukwaArchiveKey key = new ChukwaArchiveKey();
      ChunkImpl chunk = ChunkImpl.getBlankChunk();
      FileWriter out = new FileWriter(outputFile);
      try {
        while (r.next(key, chunk)) {
          out.write(new String(chunk.getData()));
        }
      } finally {
        out.close();
        r.close();
      }
  
    }
  
    public static void extractRawLogFromDump(String directory, String fileName)
        throws Exception {
      File inputFile = new File(directory + fileName + ".bin");
      File outputFile = new File(directory + fileName + ".raw");
      DataInputStream dis = new DataInputStream(new FileInputStream(inputFile));
      Chunk chunk = null;
      FileWriter out = new FileWriter(outputFile);
      boolean eof = false;
      do {
        try {
          chunk = ChunkImpl.read(dis);
          out.write(new String(chunk.getData()));
        } catch (EOFException e) {
          eof = true;
        }
  
      } while (!eof);
  
      dis.close();
      out.close();
    }
  }
  