content: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.cassandra.tools;
  
  import java.io.IOException;
  import java.io.PrintStream;
  import java.io.PrintWriter;
  import java.nio.ByteBuffer;
  import java.util.Comparator;
  import java.util.List;
  import java.util.Map;
  import java.util.concurrent.TimeUnit;
  import java.util.stream.Collectors;
  
  import com.google.common.collect.MinMaxPriorityQueue;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.CommandLineParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.commons.cli.PosixParser;
  
  import org.apache.cassandra.config.DatabaseDescriptor;
  import org.apache.cassandra.db.ClusteringComparator;
  import org.apache.cassandra.db.DecoratedKey;
  import org.apache.cassandra.db.SerializationHeader;
  import org.apache.cassandra.db.marshal.AbstractType;
  import org.apache.cassandra.db.marshal.UTF8Type;
  import org.apache.cassandra.db.rows.EncodingStats;
  import org.apache.cassandra.db.rows.Row;
  import org.apache.cassandra.db.rows.Unfiltered;
  import org.apache.cassandra.db.rows.UnfilteredRowIterator;
  import org.apache.cassandra.dht.IPartitioner;
  import org.apache.cassandra.io.compress.CompressionMetadata;
  import org.apache.cassandra.io.compress.ICompressor;
  import org.apache.cassandra.io.sstable.Descriptor;
  import org.apache.cassandra.io.sstable.ISSTableScanner;
  import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
  import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.apache.cassandra.io.sstable.format.StatsComponent;
  import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
  import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
  import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
  import org.apache.cassandra.io.util.File;
  import org.apache.cassandra.schema.TableMetadata;
  import org.apache.cassandra.schema.TableMetadataRef;
  import org.apache.cassandra.tools.Util.TermHistogram;
  import org.apache.cassandra.utils.FBUtilities;
  import org.apache.cassandra.utils.Pair;
  
  import static org.apache.cassandra.tools.Util.BLUE;
  import static org.apache.cassandra.tools.Util.CYAN;
  import static org.apache.cassandra.tools.Util.RESET;
  import static org.apache.cassandra.tools.Util.WHITE;
  import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
  import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords;
  
  /**
   * Shows the contents of sstable metadata
   */
  public class SSTableMetadataViewer
  {
      private static final Options options = new Options();
      private static CommandLine cmd;
      private static final String COLORS = "c";
      private static final String UNICODE = "u";
      private static final String GCGS_KEY = "g";
      private static final String TIMESTAMP_UNIT = "t";
      private static final String SCAN = "s";
      private static final String HELP = "h";
      private static final Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed();
  
      static
      {
          DatabaseDescriptor.toolInitialization();
      }
  
      boolean color;
      boolean unicode;
      int gc;
      PrintStream out;
      String[] files;
      TimeUnit tsUnit;
  
      public SSTableMetadataViewer()
      {
          this(true, true, 0, TimeUnit.MICROSECONDS, System.out);
      }
  
      public SSTableMetadataViewer(boolean color, boolean unicode, int gc, TimeUnit tsUnit, PrintStream out)
      {
          this.color = color;
          this.tsUnit = tsUnit;
          this.unicode = unicode;
          this.out = out;
          this.gc = gc;
      }
  
      public static String deletion(long time)
      {
          if (time == 0 || time == Long.MAX_VALUE)
          {
              return "no tombstones";
          }
          return toDateString(time, TimeUnit.SECONDS);
      }
  
      public static String toDateString(long time, TimeUnit unit)
      {
          if (time == 0)
          {
              return null;
          }
          return new java.text.SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(new java.util.Date(unit.toMillis(time)));
      }
  
      public static String toDurationString(long duration, TimeUnit unit)
      {
          if (duration == 0)
          {
              return null;
          }
          else if (duration == Integer.MAX_VALUE)
          {
              return "never";
          }
          return formatDurationWords(unit.toMillis(duration), true, true);
      }
  
      public static String toByteString(long bytes)
      {
          if (bytes == 0)
              return null;
          else if (bytes < 1024)
              return bytes + " B";
  
          int exp = (int) (Math.log(bytes) / Math.log(1024));
          char pre = "kMGTP".charAt(exp - 1);
          return String.format("%.1f %sB", bytes / Math.pow(1024, exp), pre);
      }
  
      public String scannedOverviewOutput(String key, long value)
      {
          StringBuilder sb = new StringBuilder();
          if (color) sb.append(CYAN);
          sb.append('[');
          if (color) sb.append(RESET);
          sb.append(key);
          if (color) sb.append(CYAN);
          sb.append("] ");
          if (color) sb.append(RESET);
          sb.append(value);
          return sb.toString();
      }
  
      private void printScannedOverview(Descriptor descriptor, StatsMetadata stats) throws IOException
      {
          TableMetadata cfm = Util.metadataFromSSTable(descriptor);
          SSTableReader reader = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(cfm));
          try (ISSTableScanner scanner = reader.getScanner())
          {
              long bytes = scanner.getLengthInBytes();
              MinMaxPriorityQueue<ValuedByteBuffer> widestPartitions = MinMaxPriorityQueue
                                                                       .orderedBy(VCOMP)
                                                                       .maximumSize(5)
                                                                       .create();
              MinMaxPriorityQueue<ValuedByteBuffer> largestPartitions = MinMaxPriorityQueue
                                                                        .orderedBy(VCOMP)
                                                                        .maximumSize(5)
                                                                        .create();
              MinMaxPriorityQueue<ValuedByteBuffer> mostTombstones = MinMaxPriorityQueue
                                                                     .orderedBy(VCOMP)
                                                                     .maximumSize(5)
                                                                     .create();
              long partitionCount = 0;
              long rowCount = 0;
              long tombstoneCount = 0;
              long cellCount = 0;
              double totalCells = stats.totalColumnsSet;
              int lastPercent = 0;
              long lastPercentTime = 0;
              while (scanner.hasNext())
              {
                  try (UnfilteredRowIterator partition = scanner.next())
                  {
  
                      long psize = 0;
                      long pcount = 0;
                      int ptombcount = 0;
                      partitionCount++;
                      if (!partition.staticRow().isEmpty())
                      {
                          rowCount++;
                          pcount++;
                          psize += partition.staticRow().dataSize();
                      }
                      if (!partition.partitionLevelDeletion().isLive())
                      {
                          tombstoneCount++;
                          ptombcount++;
                      }
                      while (partition.hasNext())
                      {
                          Unfiltered unfiltered = partition.next();
                          switch (unfiltered.kind())
                          {
                              case ROW:
                                  rowCount++;
                                  Row row = (Row) unfiltered;
                                  psize += row.dataSize();
                                  pcount++;
                                  for (org.apache.cassandra.db.rows.Cell<?> cell : row.cells())
                                  {
                                      cellCount++;
                                      double percentComplete = Math.min(1.0, cellCount / totalCells);
                                      if (lastPercent != (int) (percentComplete * 100) &&
                                          (currentTimeMillis() - lastPercentTime) > 1000)
                                      {
                                          lastPercentTime = currentTimeMillis();
                                          lastPercent = (int) (percentComplete * 100);
                                          if (color)
                                              out.printf("\r%sAnalyzing SSTable...  %s%s %s(%%%s)", BLUE, CYAN,
                                                         Util.progress(percentComplete, 30, unicode),
                                                         RESET,
                                                         (int) (percentComplete * 100));
                                          else
                                              out.printf("\rAnalyzing SSTable...  %s (%%%s)",
                                                         Util.progress(percentComplete, 30, unicode),
                                                         (int) (percentComplete * 100));
                                          out.flush();
                                      }
                                      if (cell.isTombstone())
                                      {
                                          tombstoneCount++;
                                          ptombcount++;
                                      }
                                  }
                                  break;
                              case RANGE_TOMBSTONE_MARKER:
                                  tombstoneCount++;
                                  ptombcount++;
                                  break;
                          }
                      }
  
                      widestPartitions.add(new ValuedByteBuffer(partition.partitionKey().getKey(), pcount));
                      largestPartitions.add(new ValuedByteBuffer(partition.partitionKey().getKey(), psize));
                      mostTombstones.add(new ValuedByteBuffer(partition.partitionKey().getKey(), ptombcount));
                  }
              }
  
              out.printf("\r%80s\r", " ");
              field("Size", bytes);
              field("Partitions", partitionCount);
              field("Rows", rowCount);
              field("Tombstones", tombstoneCount);
              field("Cells", cellCount);
              field("Widest Partitions", "");
              Util.iterToStream(widestPartitions.iterator()).sorted(VCOMP).forEach(p ->
                                                                                   {
                                                                                       out.println("  " + scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
                                                                                   });
              field("Largest Partitions", "");
              Util.iterToStream(largestPartitions.iterator()).sorted(VCOMP).forEach(p ->
                                                                                    {
                                                                                        out.print("  ");
                                                                                        out.print(scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
                                                                                        if (color)
                                                                                            out.print(WHITE);
                                                                                        out.print(" (");
                                                                                        out.print(toByteString(p.value));
                                                                                        out.print(")");
                                                                                        if (color)
                                                                                            out.print(RESET);
                                                                                        out.println();
                                                                                    });
              StringBuilder tleaders = new StringBuilder();
              Util.iterToStream(mostTombstones.iterator()).sorted(VCOMP).forEach(p ->
                                                                                 {
                                                                                     if (p.value > 0)
                                                                                     {
                                                                                         tleaders.append("  ");
                                                                                         tleaders.append(scannedOverviewOutput(cfm.partitionKeyType.getString(p.buffer), p.value));
                                                                                         tleaders.append(System.lineSeparator());
                                                                                     }
                                                                                 });
              String tombstoneLeaders = tleaders.toString();
              if (tombstoneLeaders.length() > 10)
              {
                  field("Tombstone Leaders", "");
                  out.print(tombstoneLeaders);
              }
          }
          finally
          {
              reader.selfRef().ensureReleased();
          }
      }
  
      private void printSStableMetadata(File file, boolean scan) throws IOException
      {
          Descriptor descriptor = Descriptor.fromFileWithComponent(file, false).left;
          StatsComponent statsComponent = StatsComponent.load(descriptor);
          ValidationMetadata validation = statsComponent.validationMetadata();
          StatsMetadata stats = statsComponent.statsMetadata();
          CompactionMetadata compaction = statsComponent.compactionMetadata();
          SerializationHeader.Component header = statsComponent.serializationHeader();
          Class<? extends ICompressor> compressorClass = null;
          try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor))
          {
              compressorClass = compression != null ? compression.compressor().getClass() : null;
          }
  
          field("SSTable", descriptor);
          if (scan && descriptor.version.version.compareTo("ma") >= 0)
          {
              printScannedOverview(descriptor, stats);
          }
          if (validation != null)
          {
              field("Partitioner", validation.partitioner);
              field("Bloom Filter FP chance", validation.bloomFilterFPChance);
          }
          if (stats != null)
          {
              TimeUnit tsUnit = TimeUnit.MICROSECONDS;
              field("Minimum timestamp", toDateString(stats.minTimestamp, tsUnit), Long.toString(stats.minTimestamp));
              field("Maximum timestamp", toDateString(stats.maxTimestamp, tsUnit), Long.toString(stats.maxTimestamp));
              field("Duration", durationString(stats.maxTimestamp - stats.minTimestamp));
              field("SSTable min local deletion time", deletion(stats.minLocalDeletionTime), Long.toString(stats.minLocalDeletionTime));
              field("SSTable max local deletion time", deletion(stats.maxLocalDeletionTime), Long.toString(stats.maxLocalDeletionTime));
              field("Compressor", compressorClass != null ? compressorClass.getName() : "-");
              if (compressorClass != null)
                  field("Compression ratio", stats.compressionRatio);
              field("TTL min", stats.minTTL, toDurationString(stats.minTTL, TimeUnit.SECONDS));
              field("TTL max", stats.maxTTL, toDurationString(stats.maxTTL, TimeUnit.SECONDS));
  
              if (validation != null && header != null)
                  printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType(), stats);
  
              if (header != null)
              {
                  ClusteringComparator comparator = new ClusteringComparator(header.getClusteringTypes());
                  field("Covered clusterings", stats.coveredClustering.toString(comparator));
              }
              field("Estimated droppable tombstones",
                    stats.getEstimatedDroppableTombstoneRatio((int) (currentTimeMillis() / 1000) - this.gc));
              field("SSTable Level", stats.sstableLevel);
              field("Repaired at", stats.repairedAt, toDateString(stats.repairedAt, TimeUnit.MILLISECONDS));
              field("Originating host id", stats.originatingHostId);
              field("Pending repair", stats.pendingRepair);
              field("Replay positions covered", stats.commitLogIntervals);
              field("totalColumnsSet", stats.totalColumnsSet);
              field("totalRows", stats.totalRows);
              field("Estimated tombstone drop times", "");
  
              TermHistogram estDropped = new TermHistogram(stats.estimatedTombstoneDropTime,
                                                           "Drop Time",
                                                           offset -> String.format("%d %s",
                                                                  offset,
                                                                  Util.wrapQuiet(toDateString(offset, TimeUnit.SECONDS),
                                                                                          color)),
                                                           String::valueOf);
              estDropped.printHistogram(out, color, unicode);
              field("Partition Size", "");
              TermHistogram rowSize = new TermHistogram(stats.estimatedPartitionSize,
                                                        "Size (bytes)",
                                                        offset -> String.format("%d %s",
                                                                                offset,
                                                                                Util.wrapQuiet(toByteString(offset), color)),
                                                        String::valueOf);
              rowSize.printHistogram(out, color, unicode);
              field("Column Count", "");
              TermHistogram cellCount = new TermHistogram(stats.estimatedCellPerPartitionCount,
                                                          "Columns",
                                                          String::valueOf,
                                                          String::valueOf);
              cellCount.printHistogram(out, color, unicode);
              field("Local token space coverage", stats.tokenSpaceCoverage);
          }
          if (compaction != null)
          {
              field("Estimated cardinality", compaction.cardinalityEstimator.cardinality());
          }
          if (header != null)
          {
              EncodingStats encodingStats = header.getEncodingStats();
              AbstractType<?> keyType = header.getKeyType();
              List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
              Map<ByteBuffer, AbstractType<?>> staticColumns = header.getStaticColumns();
              Map<String, String> statics = staticColumns.entrySet().stream()
                      .collect(Collectors.toMap(e -> UTF8Type.instance.getString(e.getKey()),
                                                e -> e.getValue().toString()));
              Map<ByteBuffer, AbstractType<?>> regularColumns = header.getRegularColumns();
              Map<String, String> regulars = regularColumns.entrySet().stream()
                      .collect(Collectors.toMap(e -> UTF8Type.instance.getString(e.getKey()),
                                                e -> e.getValue().toString()));
  
              field("EncodingStats minTTL", encodingStats.minTTL,
                      toDurationString(encodingStats.minTTL, TimeUnit.SECONDS));
              field("EncodingStats minLocalDeletionTime", toDateString(encodingStats.minLocalDeletionTime,
                      TimeUnit.SECONDS), Long.toString(encodingStats.minLocalDeletionTime));
              field("EncodingStats minTimestamp", toDateString(encodingStats.minTimestamp, tsUnit),
                      Long.toString(encodingStats.minTimestamp));
              field("KeyType", keyType.toString());
              field("ClusteringTypes", clusteringTypes.toString());
              field("StaticColumns", FBUtilities.toString(statics));
              field("RegularColumns", FBUtilities.toString(regulars));
              if (stats != null)
                  field("IsTransient", stats.isTransient);
          }
      }
  
      private void field(String field, Object value)
      {
          field(field, value, null);
      }
  
      private void field(String field, Object value, String comment)
      {
          StringBuilder sb = new StringBuilder();
          if (color) sb.append(BLUE);
          sb.append(field);
          if (color) sb.append(CYAN);
          sb.append(": ");
          if (color) sb.append(RESET);
          sb.append(value == null? "--" : value.toString());
  
          if (comment != null)
          {
              if (color) sb.append(WHITE);
              sb.append(" (");
              sb.append(comment);
              sb.append(')');
              if (color) sb.append(RESET);
          }
          this.out.println(sb);
      }
  
      public static String durationString(Long value)
      {
          long seconds  = TimeUnit.MICROSECONDS.toSeconds(value);
          long day = TimeUnit.SECONDS.toDays(seconds);
          long hours = TimeUnit.SECONDS.toHours(seconds) - (day * 24);
          long minute = TimeUnit.SECONDS.toMinutes(seconds) - (TimeUnit.SECONDS.toHours(seconds) * 60);
          long second = TimeUnit.SECONDS.toSeconds(seconds) - (TimeUnit.SECONDS.toMinutes(seconds) * 60);
          return "Days: " + day + " Hours: " + hours + " Minutes: " + minute + " Seconds: " + second;
      }
  
      private static void printUsage()
      {
          try (PrintWriter errWriter = new PrintWriter(System.err, true))
          {
              HelpFormatter formatter = new HelpFormatter();
              formatter.printHelp(errWriter, 120, "sstablemetadata <options> <sstable...>",
                                  String.format("%nDump information about SSTable[s] for Apache Cassandra 3.x%nOptions:"),
                                  options, 2, 1, "", true);
              errWriter.println();
          }
      }
  
      private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType<?> keyType, StatsMetadata statsMetadata)
      throws IOException
      {
          if (descriptor.version.hasKeyRange())
          {
              if (statsMetadata.firstKey == null || statsMetadata.lastKey == null)
                  return;
  
              field("First token", partitioner.getToken(statsMetadata.firstKey), keyType.getString(statsMetadata.firstKey));
              field("Last token", partitioner.getToken(statsMetadata.lastKey), keyType.getString(statsMetadata.lastKey));
          }
          else
          {
              Pair<DecoratedKey, DecoratedKey> firstLast = descriptor.getFormat().getReaderFactory().readKeyRange(descriptor, partitioner);
              if (firstLast == null)
                  return;
  
              field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey()));
              field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey()));
          }
      }
  
      /**
       * @param args
       *            a list of sstables whose metadata we're interested in
       */
      public static void main(String[] args) throws IOException
      {
          CommandLineParser parser = new PosixParser();
  
          Option disableColors = new Option(COLORS, "colors", false, "Use ANSI color sequences");
          disableColors.setOptionalArg(true);
          options.addOption(disableColors);
          Option unicode = new Option(UNICODE, "unicode", false, "Use unicode to draw histograms and progress bars");
          unicode.setOptionalArg(true);
  
          options.addOption(unicode);
          Option gcgs = new Option(GCGS_KEY, "gc_grace_seconds", true, "Time to use when calculating droppable tombstones");
          gcgs.setOptionalArg(true);
          options.addOption(gcgs);
          Option tsUnit = new Option(TIMESTAMP_UNIT, "timestamp_unit", true, "Time unit that cell timestamps are written with");
          tsUnit.setOptionalArg(true);
          options.addOption(tsUnit);
  
          Option help = new Option(HELP, "help", false, "Help");
          help.setOptionalArg(true);
          options.addOption(help);
  
          Option scanEnabled = new Option(SCAN, "scan", false,
                  "Full sstable scan for additional details. Only available in 3.0+ sstables. Defaults: false");
          scanEnabled.setOptionalArg(true);
          options.addOption(scanEnabled);
          try
          {
              cmd = parser.parse(options, args);
          }
          catch (ParseException e1)
          {
              System.err.println(e1.getMessage());
              printUsage();
              System.exit(1);
          }
  
          if (cmd.getArgs().length < 1)
          {
              System.err.println("You must supply at least one sstable");
              printUsage();
              System.exit(1);
          }
          boolean enabledColors = cmd.hasOption(COLORS);
          boolean enabledUnicode = cmd.hasOption(UNICODE);
          boolean fullScan = cmd.hasOption(SCAN);
          int gc = Integer.parseInt(cmd.getOptionValue(GCGS_KEY, "0"));
          TimeUnit ts = TimeUnit.valueOf(cmd.getOptionValue(TIMESTAMP_UNIT, "MICROSECONDS"));
          SSTableMetadataViewer metawriter = new SSTableMetadataViewer(enabledColors, enabledUnicode, gc, ts, System.out);
          for (String fname : cmd.getArgs())
          {
              File sstable = new File(fname);
              if (sstable.exists())
              {
                  metawriter.printSStableMetadata(sstable, fullScan);
              }
              else
              {
                  System.out.println("No such file: " + fname);
              }
          }
      }
  
      private static class ValuedByteBuffer
      {
          public long value;
          public ByteBuffer buffer;
  
          public ValuedByteBuffer(ByteBuffer buffer, long value)
          {
              this.value = value;
              this.buffer = buffer;
          }
  
          public long getValue()
          {
              return value;
          }
      }
  }
  