content: /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *    http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.carbondata.tool;
  
  import java.io.ByteArrayOutputStream;
  import java.io.File;
  import java.io.IOException;
  import java.io.PrintStream;
  
  import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
  import org.apache.carbondata.core.constants.CarbonVersionConstants;
  import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
  import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
  import org.apache.carbondata.core.datastore.impl.FileFactory;
  import org.apache.carbondata.core.metadata.datatype.DataTypes;
  import org.apache.carbondata.core.metadata.datatype.Field;
  import org.apache.carbondata.core.util.CarbonUtil;
  import org.apache.carbondata.core.util.path.CarbonTablePath;
  import org.apache.carbondata.sdk.file.*;
  
  import org.apache.commons.io.FileUtils;
  import org.junit.After;
  import org.junit.Assert;
  import org.junit.Before;
  import org.junit.Test;
  
  public class CarbonCliTest {
  
    private String path = "./CarbonCliTest";
    private String pathBinary = "./CarbonCliTestBinary";
  
    private String buildLines(String... lines) {
      ByteArrayOutputStream expectedOut = null;
      PrintStream expectedStream = null;
      try {
        expectedOut = new ByteArrayOutputStream();
        expectedStream = new PrintStream(expectedOut);
        for (String line : lines) {
          expectedStream.println(line);
        }
  
        return new String(expectedOut.toByteArray());
      } finally {
        CarbonUtil.closeStreams(expectedStream, expectedOut);
      }
    }
  
    @Before
    public void before() throws IOException {
      FileUtils.deleteDirectory(new File(path));
  
      Field[] fields = new Field[2];
      fields[0] = new Field("name", DataTypes.STRING);
      fields[1] = new Field("age", DataTypes.INT);
  
      TestUtil.writeFilesAndVerify(5000000, new Schema(fields), path, new String[]{"name", "age"}, 3, 8);
      TestUtil.writeFilesAndVerify(5000000, new Schema(fields), path, new String[]{"name", "age"}, 3, 8);
    }
  
    public void buildBinaryData(int rows, Schema schema, String path, String[] sortColumns,
                                int blockletSize, int blockSize)
        throws IOException, InvalidLoadOptionException {
  
      CarbonWriterBuilder builder = CarbonWriter.builder()
          .outputPath(path);
      if (sortColumns != null) {
        builder = builder.sortBy(sortColumns);
      }
      if (blockletSize != -1) {
        builder = builder.withBlockletSize(blockletSize);
      }
      if (blockSize != -1) {
        builder = builder.withBlockSize(blockSize);
      }
  
      CarbonWriter writer = builder.withCsvInput(schema).writtenBy("TestUtil").build();
  
      for (int i = 0; i < rows; i++) {
        writer.write(new String[]{
            "robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf((double) i / 2)});
      }
      for (int i = 0; i < rows; i++) {
        writer.write(new String[]{
            "robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf("robot" + i / 2)});
      }
      writer.close();
    }
  
    @Test
    public void testInvalidCmd() {
      String[] args = {"-cmd", "DD", "-p", path};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
      Assert.assertTrue(output.contains("command DD is not supported"));
  
      String[] args2 = {"-p", path};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args2, stream);
      output = new String(out.toByteArray());
      Assert.assertTrue(output.contains("Parsing failed. Reason: Missing required option: cmd"));
    }
  
    @Test
    public void testSummaryOutputIndividual() {
      String[] args = {"-cmd", "summary", "-p", path};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
  
      String expectedOutput = buildLines(
              "Input Folder: ./CarbonCliTest" ,
              "## Summary",
              "total: 6 blocks, 2 shards, 10 blocklets, 314 pages, 10,000,000 rows, 24.91MB",
              "avg: 4.15MB/block, 2.49MB/blocklet, 1,666,666 rows/block, 1,000,000 rows/blocklet");
      Assert.assertTrue(output.contains(expectedOutput));
  
      String[] args2 = {"-cmd", "summary", "-p", path, "-s"};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args2, stream);
      output = new String(out.toByteArray());
  
      expectedOutput = buildLines(
          "Column Name  Data Type  Column Type  SortColumn  Encoding  Ordinal  Id  ",
          "name         STRING     dimension    true        []        0        NA  ",
          "age          INT        dimension    true        []        1        NA  ");
      Assert.assertTrue(output.contains(expectedOutput));
  
      String[] args3 = {"-cmd", "summary", "-p", path, "-t"};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args3, stream);
      output = new String(out.toByteArray());
  
      expectedOutput = buildLines(
          "## Table Properties",
          "schema file not found");
      Assert.assertTrue(output.contains(expectedOutput));
  
      String[] args4 = {"-cmd", "summary", "-p", path, "-b", "7"};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args4, stream);
      output = new String(out.toByteArray());
  
      expectedOutput = buildLines(
          "BLK  BLKLT  NumPages  NumRows    Size    " ,
          "0    0      32        1,024,000  2.55MB  " ,
          "0    1      32        1,024,000  2.55MB  " ,
          "1    0      32        1,024,000  2.55MB  " ,
          "1    1      32        1,024,000  2.55MB  " ,
          "2    0      29        904,000    2.26MB  ");
      Assert.assertTrue(output.contains(expectedOutput));
  
      String[] args5 = {"-cmd", "summary", "-p", path, "-c", "name"};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args5, stream);
      output = new String(out.toByteArray());
  
      expectedOutput = buildLines(
          "BLK  BLKLT  Meta Size  Data Size  LocalDict  DictEntries  DictSize  AvgPageSize  Min%  Max%  Min     Max     " ,
          "0    0      2.16KB     2.46KB     true       3            22.0B     9.0B         NA    NA    robot0  robot2  " ,
          "0    1      2.16KB     2.46KB     true       3            22.0B     9.0B         NA    NA    robot2  robot4  " ,
          "1    0      2.16KB     2.46KB     true       3            22.0B     9.0B         NA    NA    robot4  robot6  " ,
          "1    1      2.16KB     2.45KB     true       3            22.0B     9.0B         NA    NA    robot6  robot8  " ,
          "2    0      1.96KB     2.22KB     true       2            18.0B     9.0B         NA    NA    robot8  robot9  ");
      Assert.assertTrue(output.contains(expectedOutput));
    }
  
    @Test
    public void testSortColumnsOfSegmentFolder() {
      String[] args = {"-cmd", "sort_columns", "-p", path};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
      String expectedOutput = buildLines(
          "Input Folder: ./CarbonCliTest",
          "sorted by name,age");
      Assert.assertTrue(output.contains(expectedOutput));
    }
  
    @Test
    public void testSummaryOutputAll() {
      String[] args = {"-cmd", "summary", "-p", path, "-a", "-c", "age"};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
  
      String expectedOutput = buildLines(
          "Input Folder: ./CarbonCliTest" ,
          "## Summary",
          "total: 6 blocks, 2 shards, 10 blocklets, 314 pages, 10,000,000 rows, 24.91MB",
          "avg: 4.15MB/block, 2.49MB/blocklet, 1,666,666 rows/block, 1,000,000 rows/blocklet");
  
      Assert.assertTrue(output.contains(expectedOutput));
  
      expectedOutput = buildLines(
          "Column Name  Data Type  Column Type  SortColumn  Encoding  Ordinal  Id  ",
          "name         STRING     dimension    true        []        0        NA  ",
          "age          INT        dimension    true        []        1        NA  ");
      Assert.assertTrue(output.contains(expectedOutput));
  
      expectedOutput = buildLines(
          "## Table Properties",
          "schema file not found");
      Assert.assertTrue(output.contains(expectedOutput));
  
      expectedOutput = buildLines(
          "BLK  BLKLT  NumPages  NumRows    Size    ",
          "0    0      32        1,024,000  2.55MB  ",
          "0    1      32        1,024,000  2.55MB  ",
          "1    0      32        1,024,000  2.55MB  ",
          "1    1      32        1,024,000  2.55MB  ");
      Assert.assertTrue(output.contains(expectedOutput));
  
      expectedOutput = buildLines(
          "BLK  BLKLT  Meta Size  Data Size  LocalDict  DictEntries  DictSize  AvgPageSize  Min%  Max%   Min  Max      " ,
          "0    0      3.59KB     2.55MB     false      0            0.0B      81.39KB      0.0   100.0  0    2999991  " ,
          "0    1      3.59KB     2.54MB     false      0            0.0B      81.30KB      0.0   100.0  3    2999993  " ,
          "1    0      3.59KB     2.55MB     false      0            0.0B      81.54KB      0.0   100.0  5    2999995  " ,
          "1    1      3.59KB     2.54MB     false      0            0.0B      81.30KB      0.0   100.0  7    2999997  " ,
          "2    0      3.25KB     2.25MB     false      0            0.0B      79.48KB      0.0   100.0  9    2999999  ");
      Assert.assertTrue(output.contains(expectedOutput));
      Assert.assertTrue(output.contains("## version Details"));
      Assert.assertTrue(output.contains("written_by  Version"));
      Assert.assertTrue(output.contains("TestUtil    "+ CarbonVersionConstants.CARBONDATA_VERSION));
    }
  
    @Test
    public void testSummaryPageMeta() {
      String[] args = { "-cmd", "summary", "-p", path, "-c", "name", "-k"};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
      String expectedOutput = buildLines(
          "Blocklet 0:",
          "Page 0 (offset 0, length 9): DataChunk2(chunk_meta:ChunkCompressionMeta(compression_codec:DEPRECATED, total_uncompressed_size:96000, total_compressed_size:9, compressor_name:snappy), rowMajor:false, data_page_length:5, rle_page_length:4, presence:PresenceMeta(represents_presence:false, present_bit_stream:00), sort_state:SORT_NATIVE, encoders:[RLE], encoder_meta:[], min_max:BlockletMinMaxIndex(min_values:[72 6F 62 6F 74 30], max_values:[72 6F 62 6F 74 30], min_max_presence:[true]), numberOfRowsInpage:32000)");
      Assert.assertTrue(output.contains(expectedOutput));
    }
  
    @Test
    public void testSummaryAllColumns() {
      String[] args = { "-cmd", "summary", "-p", path, "-C" };
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
      Assert.assertTrue(output.contains("Block  Blocklet  Column Name  Meta Size  Data Size"));
    }
  
    @Test
    public void testSummaryAllColumnsForOneFile() {
      CarbonFile folder = FileFactory.getCarbonFile(path);
      CarbonFile[] carbonFiles = folder.listFiles(new CarbonFileFilter() {
  
        @Override
        public boolean accept(CarbonFile file) {
          return file.getName().endsWith(CarbonTablePath.CARBON_DATA_EXT);
        }
      });
  
      String[] args = { "-cmd", "summary", "-p", carbonFiles[0].getCanonicalPath(), "-C" };
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
      String output = new String(out.toByteArray());
      Assert.assertTrue(output.contains("Block  Blocklet  Column Name  Meta Size  Data Size"));
    }
  
    @Test
    public void testBenchmark() {
      String[] args = {"-cmd", "benchmark", "-p", path, "-a", "-c", "name"};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
    }
  
    @Test
    public void testBinary() throws IOException, InvalidLoadOptionException {
      FileUtils.deleteDirectory(new File(pathBinary));
      Field[] fields = new Field[3];
      fields[0] = new Field("name", DataTypes.STRING);
      fields[1] = new Field("age", DataTypes.INT);
      fields[2] = new Field("binaryField", DataTypes.BINARY);
  
      buildBinaryData(5000000, new Schema(fields), pathBinary, new String[]{"name"}, 3, 8);
      String[] args = {"-cmd", "summary", "-p", pathBinary};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      PrintStream stream = new PrintStream(out);
      CarbonCli.run(args, stream);
  
      String[] args2 = {"-cmd", "summary", "-p", pathBinary, "-s"};
      out = new ByteArrayOutputStream();
      stream = new PrintStream(out);
      CarbonCli.run(args2, stream);
      String output = new String(out.toByteArray());
  
      Assert.assertTrue(output.contains("binaryfield") && output.contains("BINARY"));
      FileUtils.deleteDirectory(new File(pathBinary));
    }
  
  
    @After
    public void after() throws IOException {
      FileUtils.deleteDirectory(new File(path));
      FileUtils.deleteDirectory(new File(pathBinary));
    }
  
  }
  