content: /*
   * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
   * (the "License"). You may not use this work except in compliance with the License, which is
   * available at www.apache.org/licenses/LICENSE-2.0
   *
   * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
   * either express or implied, as more fully set forth in the License.
   *
   * See the NOTICE file distributed with this work for information regarding copyright ownership.
   */
  
  package alluxio.cli.bundler.command;
  
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertNotEquals;
  import static org.junit.Assert.assertThat;
  import static org.mockito.ArgumentMatchers.eq;
  import static org.mockito.Mockito.mock;
  import static org.mockito.Mockito.when;
  
  import alluxio.cli.bundler.InfoCollectorTestUtils;
  import alluxio.client.file.FileSystemContext;
  import alluxio.conf.Configuration;
  import alluxio.conf.PropertyKey;
  import alluxio.exception.AlluxioException;
  import alluxio.util.CommonUtils;
  
  import org.apache.commons.cli.CommandLine;
  import org.hamcrest.Description;
  import org.hamcrest.TypeSafeMatcher;
  import org.junit.After;
  import org.junit.Before;
  import org.junit.Test;
  
  import java.io.File;
  import java.io.FileWriter;
  import java.io.IOException;
  import java.time.LocalDateTime;
  import java.time.ZoneId;
  import java.time.ZoneOffset;
  import java.time.format.DateTimeFormatter;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Map;
  import java.util.Set;
  
  public class CollectLogCommandTest {
    private static final int MILLISEC_TO_NANOSEC = 1_000_000;
  
    private File mTestDir;
    private Set<String> mExpectedFiles;
  
    @Before
    public void initLogDirAndConf() throws IOException {
      mTestDir = prepareLogDir();
      Configuration.set(PropertyKey.LOGS_DIR, mTestDir.getAbsolutePath());
    }
  
    @After
    public void emptyLogDir() {
      Configuration.unset(PropertyKey.LOGS_DIR);
      mTestDir.delete();
    }
  
    // Prepare a temp dir with some log files
    private File prepareLogDir() throws IOException {
      // The dir path will contain randomness so will be different every time
      File testLogDir = InfoCollectorTestUtils.createTemporaryDirectory();
      // Prepare the normal log files that normal users will have
      for (String s : CollectLogCommand.FILE_NAMES_PREFIXES) {
        InfoCollectorTestUtils.createFileInDir(testLogDir, s);
      }
      // Create some extra log files
      InfoCollectorTestUtils.createFileInDir(testLogDir, "master.log.1");
      InfoCollectorTestUtils.createFileInDir(testLogDir, "master.log.2");
      InfoCollectorTestUtils.createFileInDir(testLogDir, "worker.log.backup");
      // Remove the user file and create a directory
      File userDir = new File(testLogDir, "user");
      if (userDir.exists()) {
        userDir.delete();
      }
      // Put logs in the user log dir
      File userLogDir = InfoCollectorTestUtils.createDirInDir(testLogDir, "user");
      InfoCollectorTestUtils.createFileInDir(userLogDir, "user_hadoop.log");
      InfoCollectorTestUtils.createFileInDir(userLogDir, "user_hadoop.out");
      InfoCollectorTestUtils.createFileInDir(userLogDir, "user_root.log");
      InfoCollectorTestUtils.createFileInDir(userLogDir, "user_root.out");
  
      // Set up expectation
      Set<String> createdFiles = InfoCollectorTestUtils
              .getAllFileNamesRelative(testLogDir, testLogDir);
      mExpectedFiles = createdFiles;
  
      return testLogDir;
    }
  
    @Test
    public void logFilesCopied() throws IOException, AlluxioException {
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
  
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{cmd.getCommandName(), targetDir.getAbsolutePath()};
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
  
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void irrelevantFileIgnored() throws Exception {
      // This file will not be copied
      // Not included in the expected set
      InfoCollectorTestUtils.createFileInDir(mTestDir, "irrelevant");
  
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{cmd.getCommandName(), targetDir.getAbsolutePath()};
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
  
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void fileNameExcluded() throws Exception {
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      when(mockCommandLine.hasOption(eq("exclude-logs"))).thenReturn(true);
      when(mockCommandLine.getOptionValue(eq("exclude-logs"))).thenReturn("master.log.1, worker");
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
      mExpectedFiles.remove("master.log.1");
      mExpectedFiles.remove("worker.log");
      mExpectedFiles.remove("worker.out");
      mExpectedFiles.remove("worker.log.backup");
  
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void fileNameAdded() throws Exception {
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log");
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log.1");
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log.2");
  
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      when(mockCommandLine.hasOption(eq("additional-logs"))).thenReturn(true);
      when(mockCommandLine.getOptionValue(eq("additional-logs"))).thenReturn("alluxio_gc");
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
      mExpectedFiles.add("alluxio_gc.log");
      mExpectedFiles.add("alluxio_gc.log.1");
      mExpectedFiles.add("alluxio_gc.log.2");
  
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void fileNameReplaced() throws Exception {
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log");
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log.1");
      InfoCollectorTestUtils.createFileInDir(mTestDir, "alluxio_gc.log.2");
  
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      when(mockCommandLine.hasOption(eq("include-logs"))).thenReturn(true);
      when(mockCommandLine.getOptionValue(eq("include-logs"))).thenReturn("alluxio_gc, master");
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
      mExpectedFiles = new HashSet<>();
      mExpectedFiles.add("alluxio_gc.log");
      mExpectedFiles.add("alluxio_gc.log.1");
      mExpectedFiles.add("alluxio_gc.log.2");
      mExpectedFiles.add("master.log");
      mExpectedFiles.add("master.log.1");
      mExpectedFiles.add("master.log.2");
      mExpectedFiles.add("master.out");
      mExpectedFiles.add("master_audit.log");
  
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void illegalSelectorCombinations() throws Exception {
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
  
      // --include-logs will replace the wanted list
      // It is ambiguous to use it with --exclude-logs
      CommandLine includeExclude = mock(CommandLine.class);
      when(includeExclude.getArgs()).thenReturn(mockArgs);
      when(includeExclude.hasOption(eq("include-logs"))).thenReturn(true);
      when(includeExclude.getOptionValue(eq("include-logs"))).thenReturn("alluxio_gc, master");
      when(includeExclude.hasOption(eq("exclude-logs"))).thenReturn(true);
      when(includeExclude.getOptionValue(eq("exclude-logs"))).thenReturn("master");
      assertNotEquals(0, cmd.run(includeExclude));
  
      // --include-logs will replace the wanted list
      // It is ambiguous to use it with --additional-logs
      CommandLine includeAddition = mock(CommandLine.class);
      when(includeAddition.getArgs()).thenReturn(mockArgs);
      when(includeAddition.hasOption(eq("include-logs"))).thenReturn(true);
      when(includeAddition.getOptionValue(eq("include-logs"))).thenReturn("alluxio_gc, master");
      when(includeAddition.hasOption(eq("additional-logs"))).thenReturn(true);
      when(includeAddition.getOptionValue(eq("additional-logs"))).thenReturn("worker");
      assertNotEquals(0, cmd.run(includeExclude));
    }
  
    @Test
    public void endTimeFilter() throws Exception {
      // Define an issue end datetime
      // We ignore logs that are created after this
      LocalDateTime issueEnd = LocalDateTime.of(2020, 7, 8, 18, 0, 0);
      DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
  
      // Other logs all end before this issue
      LocalDateTime logEnd = issueEnd.minusHours(1);
      long logEndTimestamp = logEnd.toEpochSecond(ZoneOffset.UTC);
      for (File f : CommonUtils.recursiveListLocalDir(mTestDir)) {
        f.setLastModified(logEndTimestamp);
      }
  
      // Some logs are created after this time, should be ignored
      File masterLog = new File(mTestDir, "master.log");
      String log = "2020-07-08 18:53:45,129 INFO  CopycatServer - Server started successfully!\n"
              + "2020-07-08 18:53:59,129 INFO  RaftJournalSystem - Started Raft Journal System..\n"
              + "2020-07-09 00:01:59,135 INFO  DefaultMetaMaster - Standby master with address...\n"
              + "2020-07-09 00:03:59,135 INFO  AlluxioMasterProcess - All masters started\n"
              + "2020-07-09 01:12:59,138 INFO  AbstractPrimarySelector - Primary selector..\n"
              + "2020-07-09 01:53:59,139 INFO  AbstractMaster - TableMaster: Stopped standby..";
      writeToFile(masterLog, log);
      masterLog.setLastModified(LocalDateTime.of(2020, 7, 9, 1, 53, 59)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      // Some logs overlap with the end time
      File masterLog1 = new File(mTestDir, "master.log.1");
      String log1 = "2020-07-08 16:53:44,639 INFO  BlockMasterFactory - Creating..\n"
              + "2020-07-08 16:53:44,639 INFO  TableMasterFactory - Creating..\n"
              + "2020-07-08 17:01:44,639 INFO  MetaMasterFactory - Creating..\n"
              + "2020-07-08 17:01:45,639 INFO  FileSystemMasterFactory - Creating..\n"
              + "2020-07-08 18:53:44,751 INFO  UnderDatabaseRegistry - Registered UDBs: hive,glue\n"
              + "2020-07-08 18:53:44,756 INFO  LayoutRegistry - Registered Table Layouts: hive";
      writeToFile(masterLog1, log1);
      masterLog1.setLastModified(LocalDateTime.of(2020, 7, 8, 18, 53, 45)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      // Some logs end before this end time
      File masterLog2 = new File(mTestDir, "master.log.2");
      String log2 = "2020-07-08 15:40:45,656 INFO  HdfsUnderFileSystem - Successfully..\n"
              + "2020-07-08 15:51:45,782 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 15:53:45,862 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 16:52:41,876 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 16:53:43,876 INFO  JournalStateMachine - Initialized new journal..";
      writeToFile(masterLog2, log2);
      masterLog1.setLastModified(LocalDateTime.of(2020, 7, 8, 16, 53, 44)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      when(mockCommandLine.hasOption(eq("end-time"))).thenReturn(true);
      when(mockCommandLine.getOptionValue(eq("end-time"))).thenReturn(issueEnd.format(fmt));
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
      mExpectedFiles.remove("master.log");
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    @Test
    public void startTimeFilter() throws Exception {
      // Define an issue start datetime
      // We ignore logs that are done before this
      LocalDateTime issueEnd = LocalDateTime.of(2020, 7, 8, 18, 0, 0);
      DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
  
      // Other logs all end before this issue
      LocalDateTime logEnd = issueEnd.minusHours(1);
      long logEndTimestamp = logEnd.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
      for (File f : CommonUtils.recursiveListLocalDir(mTestDir)) {
        f.setLastModified(logEndTimestamp);
      }
  
      // Some logs are created after this time, should be included
      File masterLog = new File(mTestDir, "master.log");
      String log = "2020-07-08 18:53:45,129 INFO  CopycatServer - Server started successfully!\n"
              + "2020-07-08 18:53:59,129 INFO  RaftJournalSystem - Started Raft Journal System..\n"
              + "2020-07-09 00:01:59,135 INFO  DefaultMetaMaster - Standby master with address..\n"
              + "2020-07-09 00:03:59,135 INFO  AlluxioMasterProcess - All masters started\n"
              + "2020-07-09 01:12:59,138 INFO  AbstractPrimarySelector - Primary selector..\n"
              + "2020-07-09 01:53:59,139 INFO  AbstractMaster - TableMaster: Stopped..";
      writeToFile(masterLog, log);
      masterLog.setLastModified(LocalDateTime.of(2020, 7, 9, 1, 53, 59)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      // Some logs overlap with the start time, should be included
      File masterLog1 = new File(mTestDir, "master.log.1");
      String log1 = "2020-07-08 16:53:44,639 INFO  BlockMasterFactory - Creating..\n"
              + "2020-07-08 16:53:44,639 INFO  TableMasterFactory - Creating..\n"
              + "2020-07-08 17:01:44,639 INFO  MetaMasterFactory - Creating..\n"
              + "2020-07-08 17:01:45,639 INFO  FileSystemMasterFactory - Creating..\n"
              + "2020-07-08 18:53:44,751 INFO  UnderDatabaseRegistry - Registered UDBs: hive,glue\n"
              + "2020-07-08 18:53:44,756 INFO  LayoutRegistry - Registered Table Layouts: hive";
      writeToFile(masterLog1, log1);
      masterLog1.setLastModified(LocalDateTime.of(2020, 7, 8, 18, 53, 45)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      // Some logs end before the start time, should be ignored
      File masterLog2 = new File(mTestDir, "master.log.2");
      String log2 = "2020-07-08 15:40:45,656 INFO  HdfsUnderFileSystem - Successfully..\n"
              + "2020-07-08 15:51:45,782 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 15:53:45,862 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 16:52:41,876 INFO  RocksStore - Opened rocks database under path..\n"
              + "2020-07-08 16:53:43,876 INFO  JournalStateMachine - Initialized new..";
      writeToFile(masterLog2, log2);
      masterLog2.setLastModified(LocalDateTime.of(2020, 7, 8, 16, 53, 44)
              .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
  
      // Copy
      CollectLogCommand cmd = new CollectLogCommand(FileSystemContext.create());
      File targetDir = InfoCollectorTestUtils.createTemporaryDirectory();
      CommandLine mockCommandLine = mock(CommandLine.class);
      String[] mockArgs = new String[]{
              cmd.getCommandName(),
              targetDir.getAbsolutePath()
      };
      when(mockCommandLine.getArgs()).thenReturn(mockArgs);
      when(mockCommandLine.hasOption(eq("start-time"))).thenReturn(true);
      when(mockCommandLine.getOptionValue(eq("start-time"))).thenReturn(issueEnd.format(fmt));
      int ret = cmd.run(mockCommandLine);
      assertEquals(0, ret);
  
      // Files will be copied to sub-dir of target dir
      File subDir = new File(targetDir, cmd.getCommandName());
      mExpectedFiles = new HashSet<>();
      mExpectedFiles.add("master.log");
      mExpectedFiles.add("master.log.1");
      InfoCollectorTestUtils.verifyAllFiles(subDir, mExpectedFiles);
    }
  
    /**
     * A LocalDateTime matcher for assertThat.
     * */
    class DatetimeMatcher extends TypeSafeMatcher<LocalDateTime> {
      private LocalDateTime mDatetime;
  
      DatetimeMatcher setDatetime(LocalDateTime datetime) {
        mDatetime = datetime;
        return this;
      }
  
      @Override
      protected boolean matchesSafely(LocalDateTime s) {
        return mDatetime.isEqual(s);
      }
  
      @Override
      public void describeTo(Description description) {
        description.appendText("only digits");
      }
    }
  
    @Test
    public void inferDateFromLog() throws Exception {
      // A piece of Alluxio log with default format
      String alluxioLog = "2020-03-19 11:58:10,104 WARN  Configuration - Reloaded properties\n"
              + "2020-03-19 11:58:10,106 WARN  Configuration - Loaded hostname localhost\n"
              + "2020-03-19 11:58:10,591 WARN  RetryUtils - Failed to load cluster default..";
      File alluxioLogFile = new File(mTestDir, "alluxio-worker.log");
      writeToFile(alluxioLogFile, alluxioLog);
      LocalDateTime alluxioLogDatetime = CollectLogCommand.inferFileStartTime(alluxioLogFile);
      LocalDateTime expectedDatetime = LocalDateTime.of(2020, 3, 19, 11, 58, 10,
              104 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, alluxioLogDatetime),
              alluxioLogDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // A piece of sample Yarn application log with default format
      // The first >20 lines are un-timestamped information about the job
      String yarnAppLog = "\nLogged in as: user\nApplication\nAbout\nJobs\nTools\n"
              + "Log Type: container-localizer-syslog\n\n"
              + "Log Upload Time: Mon May 18 16:11:22 +0800 2020\n\n"
              + "Log Length: 0\n\n\nLog Type: stderr\n\n"
              + "Log Upload Time: Mon May 18 16:11:22 +0800 2020\n\n"
              + "Log Length: 3616\n\n"
              + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for TERM\n"
              + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for HUP\n"
              + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for INT";
      File yarnAppLogFile = new File(mTestDir, "yarn-application.log");
      writeToFile(yarnAppLogFile, yarnAppLog);
      LocalDateTime yarnAppDatetime = CollectLogCommand.inferFileStartTime(yarnAppLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 18, 16, 11, 18);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, yarnAppDatetime),
              yarnAppDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // A piece of Yarn log with default format
      String yarnLog = "2020-05-16 02:02:25,855 INFO org.apache.hadoop.yarn.server.resourcemanager"
              + ".rmcontainer.RMContainerImpl: container_e103_1584954066020_230169_01_000004 "
              + "Container Transitioned from ALLOCATED to ACQUIRED\n"
              + "2020-05-16 02:02:25,909 INFO org.apache.hadoop.yarn.server.resourcemanager"
              + ".scheduler.AppSchedulingInfo: checking for deactivate... \n"
              + "2020-05-16 02:02:26,006 INFO org.apache.hadoop.yarn.server.resourcemanager"
              + ".rmcontainer.RMContainerImpl: container_e103_1584954066020_230168_01_000047 "
              + "Container Transitioned from ALLOCATED to ACQUIRED";
      File yarnLogFile = new File(mTestDir, "yarn-rm.log");
      writeToFile(yarnLogFile, yarnLog);
      LocalDateTime yarnDatetime = CollectLogCommand.inferFileStartTime(yarnLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 16, 2, 2, 25,
              855 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, yarnDatetime),
              yarnDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // A piece of ZK log with default format
      String zkLog = "2020-05-14 21:05:53,822 WARN org.apache.zookeeper.server.NIOServerCnxn: "
              + "caught end of stream exception\n"
              + "EndOfStreamException: Unable to read additional data from client sessionid.., "
              + "likely client has closed socket\n"
              + "\tat org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:241)\n"
              + "\tat org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory..)\n"
              + "\tat java.lang.Thread.run(Thread.java:748)\n"
              + "2020-05-14 21:05:53,823 INFO org.apache.zookeeper.server.NIOServerCnxn: "
              + "Closed socket connection for client /10.64.23.190:50120 which had sessionid..\n"
              + "2020-05-14 21:05:53,911 WARN org.apache.zookeeper.server.NIOServerCnxn: "
              + "caught end of stream exception\n"
              + "EndOfStreamException: Unable to read additional data from client sessionid.."
              + "likely client has closed socket\n";
      File zkLogFile = new File(mTestDir, "zk.log");
      writeToFile(zkLogFile, zkLog);
      LocalDateTime zkDatetime = CollectLogCommand.inferFileStartTime(zkLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 14, 21, 5, 53,
              822 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, zkDatetime),
              zkDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // A piece of sample HDFS log with default format
      String hdfsLog = "2020-05-15 22:02:27,878 INFO BlockStateChange: BLOCK* addStoredBlock: "
              + "blockMap updated: 10.64.23.184:1025 is added to blk_1126197354_52572663 size 0..\n"
              + "2020-05-15 22:02:27,878 INFO BlockStateChange: BLOCK* addStoredBlock: blockMap "
              + "updated: 10.70.22.117:1025 is added to blk_1126197354_52572663 size 0..";
      File hdfsLogFile = new File(mTestDir, "hdfs.log");
      writeToFile(hdfsLogFile, hdfsLog);
      LocalDateTime hdfsDatetime = CollectLogCommand.inferFileStartTime(hdfsLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 15, 22, 2, 27,
              878 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, hdfsDatetime),
              hdfsDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // A piece of sample Presto log wtih default format
      String prestoLog = "2020-05-16T00:00:01.059+0800\tINFO\tdispatcher-query-7960"
              + "\tio.prestosql.event.QueryMonitor\tTIMELINE: Query 20200515_155959_06700_6r6b4"
              + " :: Transaction:[4d30e960-c319-439c-84dd-022ddab6fa5e] :: elapsed 1208ms :: "
              + "planning 0ms :: waiting 0ms :: scheduling 1208ms :: running 0ms :: finishing 1208ms"
              + " :: begin 2020-05-15T23:59:59.850+08:00 :: end 2020-05-16T00:00:01.058+08:00\n"
              + "2020-05-16T00:00:03.530+0800\tINFO\tdispatcher-query-7948\t"
              + "io.prestosql.event.QueryMonitor\tTIMELINE: Query 20200515_160001_06701_6r6b4"
              + " :: Transaction:[be9b396e-6697-4ecd-9782-2ca1174c1be1] :: elapsed 2316ms"
              + " :: planning 0ms :: waiting 0ms :: scheduling 2316ms :: running 0ms"
              + " :: finishing 2316ms :: begin 2020-05-16T00:00:01.212+08:00"
              + " :: end 2020-05-16T00:00:03.528+08:00";
      File prestoLogFile = new File(mTestDir, "presto.log");
      writeToFile(prestoLogFile, prestoLog);
      LocalDateTime prestoDatetime = CollectLogCommand.inferFileStartTime(prestoLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 16, 0, 0, 1,
              59 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, prestoDatetime),
              prestoDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
  
      // ParNew/CMS GC log default format
      String gcLog = "Java HotSpot(TM) 64-Bit Server VM (25.151-b12) for linux-amd64 JRE (1.8.0), "
              + "built on Sep  5 2017 19:20:58 by \"java_re\" with gcc 4.3.0 20080428..\n"
              + "Memory: 4k page, physical 1979200k(1940096k free), swap 332799k(332799k free)\n"
              + "CommandLine flags: -XX:CMSInitiatingOccupancyFraction=65 -XX:InitialHeapSize=..\n"
              + "2020-05-07T10:01:11.409+0800: 4.304: [GC (GCLocker Initiated GC) "
              + "2020-05-07T10:01:11.410+0800: 4.304: [ParNew: 25669137K->130531K(28311552K), "
              + "0.4330954 secs] 25669137K->130531K(147849216K), 0.4332463 secs] "
              + "[Times: user=4.50 sys=0.08, real=0.44 secs] ";
      File gcLogFile = new File(mTestDir, "gc.log");
      writeToFile(gcLogFile, gcLog);
      LocalDateTime gcDatetime = CollectLogCommand.inferFileStartTime(gcLogFile);
      expectedDatetime = LocalDateTime.of(2020, 5, 7, 10, 1, 11,
              409 * MILLISEC_TO_NANOSEC);
      assertThat(String.format("Expected datetime is %s but inferred %s from file%n",
              expectedDatetime, gcDatetime),
              gcDatetime,
              new DatetimeMatcher().setDatetime(expectedDatetime));
    }
  
    @Test
    public void commandLineDatetime() throws Exception {
      Map<String, LocalDateTime> stringToDatetime = new HashMap<>();
      stringToDatetime.put("2020-06-27 11:58:53,084",
              LocalDateTime.of(2020, 6, 27, 11, 58, 53, 84 * MILLISEC_TO_NANOSEC));
      stringToDatetime.put("2020-06-27 11:58:53",
              LocalDateTime.of(2020, 6, 27, 11, 58, 53));
      stringToDatetime.put("2020-06-27 11:58",
              LocalDateTime.of(2020, 6, 27, 11, 58));
      stringToDatetime.put("20/06/27 11:58:53",
              LocalDateTime.of(2020, 6, 27, 11, 58, 53));
      stringToDatetime.put("20/06/27 11:58",
              LocalDateTime.of(2020, 6, 27, 11, 58));
      stringToDatetime.put("2020-06-27T11:58:53.084+0800",
              LocalDateTime.of(2020, 6, 27, 11, 58, 53, 84 * MILLISEC_TO_NANOSEC));
      stringToDatetime.put("2020-06-27T11:58:53",
              LocalDateTime.of(2020, 6, 27, 11, 58, 53));
      stringToDatetime.put("2020-06-27T11:58",
              LocalDateTime.of(2020, 6, 27, 11, 58));
  
      for (Map.Entry<String, LocalDateTime> entry : stringToDatetime.entrySet()) {
        String key = entry.getKey();
        LocalDateTime expectedDatetime = entry.getValue();
        LocalDateTime parsedDatetime = CollectLogCommand.parseDateTime(key);
        assertThat(String.format("Expected datetime is %s but parsed %s from string %s%n",
                expectedDatetime, parsedDatetime, key),
                parsedDatetime,
                new DatetimeMatcher().setDatetime(expectedDatetime));
      }
    }
  
    private void writeToFile(File f, String content) throws IOException {
      try (FileWriter writer = new FileWriter(f)) {
        writer.write(content);
      }
    }
  }
  