content: /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *    http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.carbondata.core.util;
  
  import java.io.BufferedReader;
  import java.io.DataInputStream;
  import java.io.File;
  import java.io.FileNotFoundException;
  import java.io.FileReader;
  import java.io.FileWriter;
  import java.io.IOException;
  import java.nio.ByteBuffer;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.List;
  import java.util.Map;
  
  import org.apache.carbondata.core.datastore.block.TableBlockInfo;
  import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
  import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
  import org.apache.carbondata.core.datastore.impl.FileFactory;
  import org.apache.carbondata.core.index.Segment;
  import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
  import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
  import org.apache.carbondata.core.metadata.datatype.DataType;
  import org.apache.carbondata.core.metadata.datatype.DataTypes;
  import org.apache.carbondata.core.metadata.encoder.Encoding;
  import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
  import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
  import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
  import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
  import org.apache.carbondata.core.scan.model.ProjectionDimension;
  import org.apache.carbondata.core.util.path.CarbonTablePath;
  
  import mockit.Mock;
  import mockit.MockUp;
  import org.apache.hadoop.security.UserGroupInformation;
  import org.junit.AfterClass;
  import org.junit.Assert;
  import org.junit.BeforeClass;
  import org.junit.Test;
  
  import static junit.framework.TestCase.assertEquals;
  import static junit.framework.TestCase.assertTrue;
  import static org.hamcrest.CoreMatchers.equalTo;
  import static org.hamcrest.CoreMatchers.is;
  import static org.hamcrest.MatcherAssert.assertThat;
  
  public class CarbonUtilTest {
  
    @BeforeClass public static void setUp() throws Exception {
      new File("../core/src/test/resources/testFile.txt").createNewFile();
      new File("../core/src/test/resources/testDatabase").mkdirs();
    }
  
    @Test public void testGetBitLengthForDimensionGiveProperValue() {
      int[] cardinality = { 200, 1, 10000, 1, 10, 3 };
      int[] dimensionBitLength =
          CarbonUtil.getDimensionBitLength(cardinality, new int[] { 1, 1, 3, 1 });
      int[] expectedOutPut = { 8, 8, 14, 2, 8, 8 };
      for (int i = 0; i < dimensionBitLength.length; i++) {
        assertEquals(expectedOutPut[i], dimensionBitLength[i]);
      }
    }
  
    @Test(expected = IOException.class) public void testCloseStreams() throws IOException {
      FileReader stream = new FileReader("../core/src/test/resources/testFile.txt");
      BufferedReader br = new BufferedReader(stream);
      CarbonUtil.closeStreams(br);
      br.ready();
    }
  
    @Test public void testToGetCardinalityForPerIncrLessThan0() {
      int result = CarbonUtil.getIncrementedCardinality(6);
      assertEquals(result, 3);
    }
  
    @Test public void testToGetCardinality() {
      int result = CarbonUtil.getIncrementedCardinality(10);
      assertEquals(result, 4);
    }
  
    @Test public void testToGetCardinalityForArray() {
      int[] cardinality = { 10, 20, 0, 6 };
      int[] actualResult = CarbonUtil.getIncrementedCardinality(cardinality);
      int[] expectedResult = { 4, 5, 1, 3 };
      for (int i = 0; i < cardinality.length; i++) {
        assertEquals(actualResult[i], expectedResult[i]);
      }
    }
  
    @Test public void testToGetIncrementedCardinalityFullyFilled() {
      int[] cardinality = { 200, 20, 0, 10 };
      int[] actualResult = CarbonUtil.getIncrementedCardinalityFullyFilled(cardinality);
      int[] expectedResult = { 8, 8, 64, 8 };
      for (int i = 0; i < cardinality.length; i++) {
        assertEquals(actualResult[i], expectedResult[i]);
      }
    }
  
    @Test public void testToDeleteFolderForValidPath()
        throws InterruptedException, IOException {
      File testDir = new File("../core/src/test/resources/testDir");
      testDir.mkdirs();
      CarbonUtil.deleteFoldersAndFiles(testDir);
      assertTrue(!testDir.isDirectory());
    }
  
    @Test(expected = IOException.class) public void testToDeleteFolderWithIOException()
        throws InterruptedException, IOException {
      File testDir = new File("../core/src/test/resources/testDir");
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws IOException {
          throw new IOException();
        }
      };
      CarbonUtil.deleteFoldersAndFiles(testDir);
    }
  
    @Test(expected = InterruptedException.class)
    public void testToDeleteFolderWithInterruptedException()
        throws InterruptedException, IOException {
      File testDir = new File("../core/src/test/resources/testDir");
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws InterruptedException {
          throw new InterruptedException();
        }
      };
      CarbonUtil.deleteFoldersAndFiles(testDir);
    }
  
    @Test public void testToDeleteFileForValidPath()
        throws InterruptedException, IOException {
      File testDir = new File("../core/src/test/resources/testDir/testFile.csv");
      testDir.mkdirs();
      CarbonUtil.deleteFoldersAndFiles(testDir);
      assertTrue(!testDir.isFile());
    }
  
    @Test public void testToDeleteFoldersAndFilesForValidFolder()
        throws InterruptedException, IOException {
      String folderPath = "../core/src/test/resources/testDir/carbonDir";
      new File(folderPath).mkdirs();
      LocalCarbonFile testDir = new LocalCarbonFile(folderPath);
      CarbonUtil.deleteFoldersAndFiles(testDir);
      assertTrue(!testDir.exists());
    }
  
    @Test(expected = IOException.class) public void testToDeleteFoldersAndFilesWithIOException()
        throws InterruptedException, IOException {
      LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir/carbonDir");
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws IOException {
          throw new IOException();
        }
      };
      CarbonUtil.deleteFoldersAndFiles(testDir);
    }
  
    @Test(expected = InterruptedException.class) public void testToDeleteFoldersAndFilesWithInterruptedException()
        throws InterruptedException, IOException {
      LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir/carbonDir");
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws InterruptedException {
          throw new InterruptedException();
        }
      };
      CarbonUtil.deleteFoldersAndFiles(testDir);
    }
  
    @Test public void testToDeleteFoldersAndFilesForValidCarbonFile()
        throws InterruptedException, IOException {
      LocalCarbonFile testDir =
          new LocalCarbonFile("../core/src/test/resources/testDir/testCarbonFile");
      testDir.createNewFile();
      CarbonUtil.deleteFoldersAndFiles(testDir);
      assertTrue(!testDir.exists());
    }
  
    @Test public void testToDeleteFoldersAndFilesForCarbonFileSilently()
        throws IOException, InterruptedException {
      LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir");
      testDir.createNewFile();
      CarbonUtil.deleteFoldersAndFilesSilent(testDir);
      assertTrue(!testDir.exists());
    }
  
    @Test(expected = IOException.class)
    public void testToDeleteFoldersAndFilesSintlyWithIOException()
        throws IOException, InterruptedException {
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws IOException {
          throw new IOException();
        }
      };
      LocalCarbonFile testDir =
          new LocalCarbonFile("../unibi-solutions/system/carbon/badRecords/badLogPath");
      CarbonUtil.deleteFoldersAndFilesSilent(testDir);
    }
  
    @Test(expected = InterruptedException.class)
    public void testToDeleteFoldersAndFilesSintlyWithInterruptedException()
        throws IOException, InterruptedException {
      new MockUp<UserGroupInformation>() {
        @SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
            throws InterruptedException {
          throw new InterruptedException();
        }
      };
      LocalCarbonFile testDir =
          new LocalCarbonFile("../unibi-solutions/system/carbon/badRecords/badLogPath");
      CarbonUtil.deleteFoldersAndFilesSilent(testDir);
    }
  
    @Test public void testToDeleteFiles() throws IOException {
      String baseDirectory = "../core/src/test/resources/";
      File file1 = new File(baseDirectory + "File1.txt");
      File file2 = new File(baseDirectory + "File2.txt");
      file1.createNewFile();
      file2.createNewFile();
      File[] files = { file1, file2 };
      CarbonUtil.deleteFiles(files);
      assertTrue(!file1.exists());
      assertTrue(!file2.exists());
    }
  
    @Test public void testToGetNextLesserValue() {
      byte[] dataChunks = { 5, 6, 7, 8, 9 };
      byte[] compareValues = { 7 };
      FixedLengthDimensionColumnPage fixedLengthDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
      int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
      assertEquals(result, 1);
    }
  
    @Test public void testToGetNextLesserValueToTarget() {
      byte[] dataChunks = { 7, 7, 7, 8, 9 };
      byte[] compareValues = { 7 };
      FixedLengthDimensionColumnPage fixedLengthDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
      int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
      assertEquals(result, -1);
    }
  
    @Test public void testToGetnextGreaterValue() {
      byte[] dataChunks = { 5, 6, 7, 8, 9 };
      byte[] compareValues = { 7 };
      FixedLengthDimensionColumnPage fixedLengthDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
      int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
      assertEquals(result, 3);
    }
  
    @Test public void testToConvertToIntegerList() {
      int[] integerArray = { 10, 20, 30, 40 };
      List<Integer> integerList = CarbonUtil.convertToIntegerList(integerArray);
      for (int i = 0; i < integerArray.length; i++) {
        assertEquals(integerArray[i], (int) integerList.get(i));
      }
    }
  
    @Test public void testToGetnextGreaterValueToTarget() {
      byte[] dataChunks = { 5, 6, 7, 7, 7 };
      byte[] compareValues = { 7 };
      FixedLengthDimensionColumnPage fixedLengthDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
      int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
      assertEquals(result, 5);
    }
  
    @Test public void testToUnescapeChar() {
      String[] input = { "\\001", "\\t", "\\r", "\\b", "\\n", "\\f", "\\r\\n", "\\\\" };
      String[] output = { "\001", "\t", "\r", "\b", "\n", "\f", "\r\n", "\\" };
      for (int i = 0; i < input.length; i++) {
        assertEquals(CarbonUtil.unescapeChar(input[i]), output[i]);
      }
    }
  
    @Test public void testForDelimiterConverter() {
      String[] input =
          { "|", "*", ".", ":", "^", "\\", "$", "+", "?", "(", ")", "{", "}", "[", "]", "'" };
      String[] expectedResult =
          { "\\|", "\\*", "\\.", "\\:", "\\^", "\\\\", "\\$", "\\+", "\\?", "\\(", "\\)", "\\{",
              "\\}", "\\[", "\\]", "'" };
      for (int i = 0; i < input.length; i++) {
        assertEquals(CarbonUtil.delimiterConverter(input[i]), expectedResult[i]);
      }
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithNoBlackSlash() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.LOCAL;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "BASE_URL";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithBlackSlash() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.LOCAL;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "BASE_URL/";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithNull() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.LOCAL;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return null;
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:////../core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithHdfs() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.HDFS;
        }
      };
      new MockUp<org.apache.hadoop.conf.Configuration>() {
        @SuppressWarnings("unused") @Mock public String get(String name) {
          return "hdfs://";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("hdfs://ha/core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "hdfs://ha/core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashLocal() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.LOCAL;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "/opt/";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:////opt/core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashHDFS() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.HDFS;
        }
      };
      new MockUp<org.apache.hadoop.conf.Configuration>() {
        @SuppressWarnings("unused") @Mock public String get(String name) {
          return "hdfs://";
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "/opt/";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "hdfs:///opt/core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithBaseURLPrefix() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.HDFS;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "hdfs://ha/opt/";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "hdfs://ha/opt/core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithBaseURLFile() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
          return FileFactory.FileType.HDFS;
        }
      };
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public String getProperty(String key) {
          return "file:///";
        }
      };
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
    }
  
    @Test public void testToCheckAndAppendHDFSUrlWithFilepathPrefix() {
      String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("file:///core/src/test/resources/testDatabase");
      assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
    }
  
    @Test public void testForisFileExists() {
      assertTrue(CarbonUtil.isFileExists("../core/src/test/resources/testFile.txt"));
    }
  
    @Test public void testForisFileExistsWithException() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
            throws IOException {
          throw new IOException();
        }
      };
      assertTrue(!CarbonUtil.isFileExists("../core/src/test/resources/testFile.txt"));
    }
  
    @Test public void testToCheckAndCreateFolder() {
      boolean exists = CarbonUtil.checkAndCreateFolder("../core/src/test/resources/testDatabase");
      boolean created = CarbonUtil.checkAndCreateFolder("../core/src/test/resources/newDatabase");
      assertTrue(exists);
      assertTrue(created);
    }
  
    @Test public void testToCheckAndCreateFolderWithException() {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
            throws IOException {
          throw new IOException();
        }
      };
      boolean exists = CarbonUtil.checkAndCreateFolder("../core/src/test/resources/testDatabase1");
      assertTrue(!exists);
    }
  
    @Test public void testToGetFileSize() {
      assertEquals(CarbonUtil.getFileSize("../core/src/test/resources/testFile.txt"), 0);
    }
  
    @Test public void testForHasEncoding() {
      List<Encoding> encodingList = new ArrayList<>();
      encodingList.add(Encoding.INVERTED_INDEX);
      encodingList.add(Encoding.DICTIONARY);
      encodingList.add(Encoding.DELTA);
      assertTrue(CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY));
      assertTrue(!CarbonUtil.hasEncoding(encodingList, Encoding.BIT_PACKED));
    }
  
    @Test public void testForHasDataTypes() {
      DataType[] dataTypes = {DataTypes.createDefaultDecimalType(), DataTypes.BOOLEAN, DataTypes.INT };
      assertTrue(CarbonUtil.hasDataType(DataTypes.BOOLEAN, dataTypes));
      assertTrue(!CarbonUtil.hasDataType(DataTypes.DATE, dataTypes));
    }
  
    @Test public void testForHasComplexDataTypes() {
      assertTrue(DataTypes.createDefaultArrayType().isComplexType());
      assertTrue(!DataTypes.DATE.isComplexType());
    }
  
    @Test public void testToGetComplexDataTypeArray() {
      ColumnSchema column1Schema = new ColumnSchema();
      ColumnSchema column2Schema = new ColumnSchema();
      column1Schema.setColumnName("Column1");
      column1Schema.setDataType(DataTypes.DATE);
      ProjectionDimension
          column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
  
      column2Schema.setColumnName("Column2");
      column2Schema.setDataType(DataTypes.createDefaultArrayType());
      ProjectionDimension
          column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
  
      ProjectionDimension[] queryDimensions = { column1, column2 };
  
      boolean[] dictionaryEncoding = CarbonUtil.getComplexDataTypeArray(queryDimensions);
      boolean[] expectedDictionaryEncoding = { false, true };
      for (int i = 0; i < dictionaryEncoding.length; i++) {
        assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
      }
    }
  
    @Test public void testToReadMetadataFile() throws IOException {
      new MockUp<DataFileFooterConverterV3>() {
        @SuppressWarnings("unused") @Mock
        public DataFileFooter readDataFileFooter(TableBlockInfo info) {
          DataFileFooter fileFooter = new DataFileFooter();
          fileFooter.setVersionId(ColumnarFormatVersion.V3);
          return fileFooter;
        }
      };
      TableBlockInfo info =
          new TableBlockInfo("file:/", 1, "0", new String[0], 1, ColumnarFormatVersion.V3, null);
  
      assertEquals(CarbonUtil.readMetadataFile(info).getVersionId().number(), 3);
    }
  
    @Test(expected = IOException.class)
    public void testToReadMetadataFileWithException()
        throws Exception {
      TableBlockInfo info =
          new TableBlockInfo("file:/", 1, "0", new String[0], 1, ColumnarFormatVersion.V3, null);
      CarbonUtil.readMetadataFile(info);
    }
  
    @Test public void testToFindDimension() {
      ColumnSchema column1Schema = new ColumnSchema();
      ColumnSchema column2Schema = new ColumnSchema();
      column1Schema.setColumnName("Column1");
      column2Schema.setColumnName("Column2");
      List<CarbonDimension> carbonDimension = new ArrayList<>();
      carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1));
      carbonDimension.add(new CarbonDimension(column2Schema, 2, 1, 1));
      assertEquals(CarbonUtil.findDimension(carbonDimension, "Column1"),
          new CarbonDimension(column1Schema, 1, 1, 1));
    }
  
    @Test public void testToGetColumnSchemaList() {
      ColumnSchema column1Schema = new ColumnSchema();
      ColumnSchema column2Schema = new ColumnSchema();
      column1Schema.setColumnName("Column1");
      column2Schema.setColumnName("Column2");
      List<CarbonDimension> carbonDimension = new ArrayList<>();
      carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1));
      carbonDimension.add(new CarbonDimension(column2Schema, 2, 2, 1));
  
      List<CarbonMeasure> carbonMeasure = new ArrayList<>();
      carbonMeasure.add(new CarbonMeasure(column1Schema, 1));
      carbonMeasure.add(new CarbonMeasure(column2Schema, 2));
  
      List<ColumnSchema> columnSchema =
          CarbonUtil.getColumnSchemaList(carbonDimension, carbonMeasure);
      for (int i = 0; i < carbonMeasure.size(); i++) {
        assertEquals(columnSchema.get(i), carbonMeasure.get(i).getColumnSchema());
      }
    }
  
    @Test public void testToReadHeader() throws IOException {
      File file = new File("../core/src/test/resources/sampleCSV.csv");
      FileWriter writer = new FileWriter(file);
      writer.write("id,name");
      writer.flush();
      String headers = CarbonUtil.readHeader("../core/src/test/resources/sampleCSV.csv");
      assertEquals(headers, "id,name");
      file.deleteOnExit();
    }
  
    @Test(expected = IOException.class)
    public void testToReadHeaderWithFileNotFoundException() throws IOException {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock
        public DataInputStream getDataInputStream(String path)
            throws FileNotFoundException {
          throw new FileNotFoundException();
        }
      };
      String result = CarbonUtil.readHeader("../core/src/test/resources/sampleCSV");
      assertEquals(null, result);
    }
  
    @Test(expected = IOException.class)
    public void testToReadHeaderWithIOException() throws IOException {
      new MockUp<FileFactory>() {
        @SuppressWarnings("unused") @Mock
        public DataInputStream getDataInputStream(String path)
            throws IOException {
          throw new IOException();
        }
      };
      String result = CarbonUtil.readHeader("../core/src/test/resources/sampleCSV.csv");
      assertEquals(null, result);
    }
  
    @Test public void testToPrintLine() {
      String line = CarbonUtil.printLine("*", 2);
      assertEquals(line, "**");
    }
  
    @Test public void testToGetSegmentString() {
      List<Segment> list = new ArrayList<>();
      list.add(new Segment("1", null, null));
      list.add(new Segment("2", null, null));
      String segments = CarbonUtil.convertToString(list);
      assertEquals(segments, "1,2");
    }
  
    @Test public void testToGetSegmentStringWithEmptySegmentList() {
      List<Segment> list = new ArrayList<>();
      String segments = CarbonUtil.convertToString(list);
      assertEquals(segments, "");
    }
  
    @Test public void testToGetSurrogateKey() {
      byte[] data = { 1, 1 };
      ByteBuffer byteBuffer = ByteBuffer.allocate(8);
      int a = CarbonUtil.getSurrogateKey(data, byteBuffer);
      assertEquals(a, 257);
    }
  
    @Test public void testToGetDictionaryChunkSize() {
      new MockUp<CarbonProperties>() {
        @SuppressWarnings("unused") @Mock public CarbonProperties getInstance()
            throws NumberFormatException {
          throw new NumberFormatException();
        }
      };
      int expectedResult = CarbonUtil.getDictionaryChunkSize();
      assertEquals(expectedResult, 10000);
    }
  
    @Test public void testToPackByteBufferIntoSingleByteArrayWithNull() {
      byte[] byteArray = CarbonUtil.packByteBufferIntoSingleByteArray(null);
      assertEquals(null, byteArray);
    }
  
    @Test public void testToPackByteBufferIntoSingleByteArray() {
      ByteBuffer[] byteBuffers = { ByteBuffer.allocate(1), ByteBuffer.allocate(2) };
      byte[] byteArray = CarbonUtil.packByteBufferIntoSingleByteArray(byteBuffers);
      byte[] expectedResult = { 0, 4, 0, 5, 0, 0, 0, 0 };
      for (int i = 0; i < byteArray.length; i++) {
        assertEquals(expectedResult[i], byteArray[i]);
      }
    }
  
    @Test public void testToIdentifyDimensionType() {
      ColumnSchema column1Schema = new ColumnSchema();
      ColumnSchema column2Schema = new ColumnSchema();
      ColumnSchema column3Schema = new ColumnSchema();
      column1Schema.setColumnName("Column1");
      column1Schema.setDataType(DataTypes.DATE);
      column1Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
      column2Schema.setColumnName("Column2");
      column2Schema.setDataType(DataTypes.DATE);
      column2Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
      column3Schema.setColumnName("Column3");
      column3Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.INVERTED_INDEX));
      CarbonDimension carbonDimension = new CarbonDimension(column1Schema, 1, 1, 1);
      CarbonDimension carbonDimension2 = new CarbonDimension(column2Schema, 2, 2, 2);
      CarbonDimension carbonDimension3 = new CarbonDimension(column3Schema, 3, 3, 3);
      List<CarbonDimension> carbonDimensions =
          Arrays.asList(carbonDimension, carbonDimension2, carbonDimension3);
      boolean[] result = CarbonUtil.identifyDimensionType(carbonDimensions);
      assertThat(result, is(equalTo(new boolean[] { true, true, false })));
    }
  
    @Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo1() {
      byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
      byte[] compareValue = { 5 };
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
      int result = CarbonUtil
          .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
      assertEquals(-2, result);
    }
  
    @Test public void testToGetFirstIndexUsingBinarySearchWithCompareToLessThan0() {
      byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
      byte[] compareValue = { 30 };
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
      int result = CarbonUtil
          .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
      assertEquals(2, result);
    }
  
    @Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo0() {
      byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
      byte[] compareValue = { 10 };
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
      int result = CarbonUtil
          .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
      assertEquals(0, result);
    }
  
    @Test public void testToGetFirstIndexUsingBinarySearchWithMatchUpLimitTrue() {
      byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
      byte[] compareValue = { 10 };
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
          new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
      int result = CarbonUtil
          .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true);
      assertEquals(2, result);
    }
  
    @Test
    public void testBinaryRangeSearch() {
  
      byte[] dataChunk = new byte[10];
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
      byte[] keyWord = new byte[1];
      int[] range;
  
      dataChunk = "abbcccddddeffgggh".getBytes();
      byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
        dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
      }
  
      keyWord[0] = Byte.valueOf("97");
      int[] expectRangeIndex = new int[2];
      expectRangeIndex[0] = 0;
      expectRangeIndex[1] = 0;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      keyWord[0] = Byte.valueOf("104");
      expectRangeIndex = new int[2];
      expectRangeIndex[0] = 16;
      expectRangeIndex[1] = 16;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      keyWord[0] = Byte.valueOf("101");
      expectRangeIndex = new int[2];
      expectRangeIndex[0] = 10;
      expectRangeIndex[1] = 10;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      keyWord[0] = Byte.valueOf("99");
      expectRangeIndex = new int[2];
      expectRangeIndex[0] = 3;
      expectRangeIndex[1] = 5;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      dataChunk = "ab".getBytes();
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      keyWord[0] = Byte.valueOf("97");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(0, range[0]);
      assertEquals(0, range[1]);
  
      keyWord[0] = Byte.valueOf("98");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(1, range[0]);
      assertEquals(1, range[1]);
  
      dataChunk = "aabb".getBytes();
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      keyWord[0] = Byte.valueOf("97");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(0, range[0]);
      assertEquals(1, range[1]);
  
      keyWord[0] = Byte.valueOf("98");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(2, range[0]);
      assertEquals(3, range[1]);
  
      dataChunk = "a".getBytes();
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      keyWord[0] = Byte.valueOf("97");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(0, range[0]);
      assertEquals(0, range[1]);
  
      dataChunk = "aa".getBytes();
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      keyWord[0] = Byte.valueOf("97");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(0, range[0]);
      assertEquals(1, range[1]);
  
      dataChunk = "aabbbbbbbbbbcc".getBytes();
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
      keyWord[0] = Byte.valueOf("98");
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
      assertEquals(2, range[0]);
      assertEquals(11, range[1]);
  
    }
  
    @Test
    public void IndexUsingBinarySearchLengthTwo() {
  
      byte[] dataChunk = new byte[10];
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
  
      byte[] keyWord = new byte[2];
  
      dataChunk = "aabbbbbbbbbbcc".getBytes();
      byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
  
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
        dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
      }
  
      keyWord[0] = Byte.valueOf("98");
      keyWord[1] = Byte.valueOf("98");
      int[] expectRangeIndex = new int[2];
      expectRangeIndex[0] = 1;
      expectRangeIndex[1] = 5;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      keyWord[0] = Byte.valueOf("97");
      keyWord[1] = Byte.valueOf("97");
  
      expectRangeIndex = new int[2];
      expectRangeIndex[0] = 0;
      expectRangeIndex[1] = 0;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
      keyWord[0] = Byte.valueOf("99");
      keyWord[1] = Byte.valueOf("99");
      expectRangeIndex = new int[2];
      expectRangeIndex[0] = 6;
      expectRangeIndex[1] = 6;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
    }
  
    @Test
    public void IndexUsingBinarySearchLengthThree() {
  
      byte[] dataChunk = new byte[10];
      FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
  
      byte[] keyWord = new byte[3];
  
      dataChunk = "aaabbbbbbbbbccc".getBytes();
      byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
  
      fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
          dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
  
      for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
        dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
      }
  
      keyWord[0] = Byte.valueOf("98");
      keyWord[1] = Byte.valueOf("98");
      keyWord[2] = Byte.valueOf("98");
      int[] expectRangeIndex = new int[2];
      expectRangeIndex[0] = 1;
      expectRangeIndex[1] = 3;
      assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
  
    }
  
    @Test
    public void testSplitSchemaStringToMapWithLessThanSplitLen() {
      String schema = generateString(399);
      Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
      Assert.assertTrue(map.size() == 2);
      String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
      Assert.assertTrue(schemaString.length() > schema.length());
    }
  
    @Test
    public void testSplitSchemaStringToMapWithEqualThanSplitLen() {
      String schema = generateString(4000);
      Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
      Assert.assertTrue(map.size() == 2);
      String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
      Assert.assertTrue(schemaString.length() > schema.length());
    }
  
    @Test
    public void testSplitSchemaStringToMapWithMoreThanSplitLen() {
      String schema = generateString(7999);
      Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
      Assert.assertTrue(map.size() == 3);
      String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
      Assert.assertTrue(schemaString.length() > schema.length());
    }
  
    @Test
    public void testSplitSchemaStringToMapWithMultiplesOfSplitLen() {
      String schema = generateString(12000);
      Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
      Assert.assertTrue(map.size() == 4);
      String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
      Assert.assertTrue(schemaString.length() > schema.length());
    }
  
    @Test
    public void testTupeIDInUpdateScenarios() {
      // Test CarbonTablePath.java
      String blockId = "Part0/Segment_0/part-0-0_batchno0-0-0-1597409791503.snappy.carbondata";
      Assert.assertEquals(CarbonTablePath.getShortBlockId(blockId), "0/0-0_0-0-0-1597409791503");
      blockId = "c3=aa/part-0-100100000100001_batchno0-0-0-1597411003332.snappy.carbondata";
      Assert.assertEquals(CarbonTablePath.getShortBlockId(blockId), "c3=aa/0-100100000100001_0-0-0-1597411003332");
      // CACHE_LEVEL = BLOCKLET case
      blockId = "Part0/Segment_0/part-0-0_batchno0-0-0-1597409791503.snappy.carbondata/0";
      Assert.assertEquals(CarbonTablePath.getShortBlockId(blockId), "0/0-0_0-0-0-1597409791503/0");
      blockId = "c3=aa/part-0-100100000100001_batchno0-0-0-1597411003332.snappy.carbondata/0";
      Assert.assertEquals(CarbonTablePath.getShortBlockId(blockId), "c3=aa/0-100100000100001_0-0-0-1597411003332/0");
      // external segment case
      blockId = "#home#root1#Projects#carbondata#integration#spark#target#warehouse#addsegtest#/Segment_2/part-0-0_batchno0-0-1-1597411388431.snappy.carbondata";
      Assert.assertEquals(CarbonTablePath.getShortBlockId(blockId), "#home#root1#Projects#carbondata#integration#spark#target#warehouse#addsegtest#/2/0-0_0-0-1-1597411388431");
      // standard table case
      String TID = "0/0-0_0-0-0-1597411901991/0/0/0";
      Assert.assertEquals(CarbonUpdateUtil.getSegmentWithBlockFromTID(TID, false), "0/0-0_0-0-0-1597411901991");
      // partition table
      TID = "c3=aa/0-100100000100001_0-0-0-1597412090158/0/0/0";
      Assert.assertEquals(CarbonUpdateUtil.getSegmentWithBlockFromTID(TID, true), "0-100100000100001_0-0-0-1597412090158");
      // external segment case
      TID = "#home#root1#Projects#carbondata#integration#spark#target#warehouse#addsegtest#/2/0-0_0-0-1-1597412329342/0/0/0";
      Assert.assertEquals(CarbonUpdateUtil.getSegmentWithBlockFromTID(TID, false), "2/0-0_0-0-1-1597412329342");
      String blockName = "part-0-0_batchno0-0-0-1597412488102.snappy.carbondata";
      // non partition table
      Assert.assertEquals(CarbonUpdateUtil.getSegmentBlockNameKey("0", blockName, false), "0/0-0_0-0-0-1597412488102");
      // partition table
      Assert.assertEquals(CarbonUpdateUtil.getSegmentBlockNameKey("0", blockName, true), "0-0_0-0-0-1597412488102");
    }
  
    @Test public void testSegmentNumberFromSegmentFile() {
      String segmentFileName = "0_1597411003332";
      Assert.assertEquals("0",
          CarbonTablePath.DataFileUtil.getSegmentNoFromSegmentFile(segmentFileName));
    }
  
    private String generateString(int length) {
      StringBuilder builder = new StringBuilder();
      for (int i = 0; i < length; i++) {
        builder.append("a");
      }
      return builder.toString();
    }
  
    private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
        FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
      int[] range;
      range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0,
          (dataChunk.length - 1) / keyWord.length, keyWord);
      assertEquals(expectRangeIndex[0], range[0]);
      assertEquals(expectRangeIndex[1], range[1]);
  
  //    int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord);
  //    assertTrue(expectRangeIndex[0] <= index && index <= range[1]);
    }
  
   	
    @AfterClass public static void testcleanUp() {
      new File("../core/src/test/resources/testFile.txt").deleteOnExit();
      new File("../core/src/test/resources/testDatabase/levelmetadata_testTable.metadata")
          .deleteOnExit();
      new File("../core/src/test/resources/testDatabase").delete();
      new File("../core/src/test/resources/newDatabase").deleteOnExit();
    }
  
  }
  