content: /*
   * Copyright 2015 Google Inc.
   *
   * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
   * in compliance with the License. You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software distributed under the License
   * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
   * or implied. See the License for the specific language governing permissions and limitations under
   * the License.
   */
  
  package com.google.googlejavaformat.java;
  
  import static com.google.common.base.Preconditions.checkNotNull;
  import static com.google.common.collect.Iterables.getLast;
  import static java.nio.charset.StandardCharsets.UTF_8;
  
  import com.google.common.base.MoreObjects;
  import com.google.common.base.Verify;
  import com.google.common.collect.DiscreteDomain;
  import com.google.common.collect.ImmutableCollection;
  import com.google.common.collect.ImmutableList;
  import com.google.common.collect.ImmutableMap;
  import com.google.common.collect.ImmutableRangeMap;
  import com.google.common.collect.ImmutableSet;
  import com.google.common.collect.Iterators;
  import com.google.common.collect.Range;
  import com.google.common.collect.RangeSet;
  import com.google.common.collect.TreeRangeSet;
  import com.google.googlejavaformat.Input;
  import com.google.googlejavaformat.Newlines;
  import com.google.googlejavaformat.java.JavacTokens.RawTok;
  import openjdk.tools.javac.file.JavacFileManager;
  import openjdk.tools.javac.parser.Tokens.TokenKind;
  import openjdk.tools.javac.tree.JCTree.JCCompilationUnit;
  import openjdk.tools.javac.util.Context;
  import openjdk.tools.javac.util.Log;
  import openjdk.tools.javac.util.Log.DeferredDiagnosticHandler;
  import openjdk.tools.javac.util.Options;
  import java.io.IOException;
  import java.net.URI;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Iterator;
  import java.util.List;
  import jdkx.tools.Diagnostic;
  import jdkx.tools.DiagnosticCollector;
  import jdkx.tools.DiagnosticListener;
  import jdkx.tools.JavaFileManager;
  import jdkx.tools.JavaFileObject;
  import jdkx.tools.JavaFileObject.Kind;
  import jdkx.tools.SimpleJavaFileObject;
  
  /** {@code JavaInput} extends {@link Input} to represent a Java input document. */
  public final class JavaInput extends Input {
    /**
     * A {@code JavaInput} is a sequence of {@link Tok}s that cover the Java input. A {@link Tok} is
     * either a token (if {@code isToken()}), or a non-token, which is a comment (if {@code
     * isComment()}) or a newline (if {@code isNewline()}) or a maximal sequence of other whitespace
     * characters (if {@code isSpaces()}). Each {@link Tok} contains a sequence of characters, an
     * index (sequential starting at {@code 0} for tokens and comments, else {@code -1}), and a
     * ({@code 0}-origin) position in the input. The concatenation of the texts of all the {@link
     * Tok}s equals the input. Each Input ends with a token EOF {@link Tok}, with empty text.
     *
     * <p>A {@code /*} comment possibly contains newlines; a {@code //} comment does not contain the
     * terminating newline character, but is followed by a newline {@link Tok}.
     */
    static final class Tok implements Input.Tok {
      private final int index;
      private final String originalText;
      private final String text;
      private final int position;
      private final int columnI;
      private final boolean isToken;
      private final TokenKind kind;
  
      /**
       * The {@code Tok} constructor.
       *
       * @param index its index
       * @param originalText its original text, before removing Unicode escapes
       * @param text its text after removing Unicode escapes
       * @param position its {@code 0}-origin position in the input
       * @param columnI its {@code 0}-origin column number in the input
       * @param isToken whether the {@code Tok} is a token
       * @param kind the token kind
       */
      Tok(
          int index,
          String originalText,
          String text,
          int position,
          int columnI,
          boolean isToken,
          TokenKind kind) {
        this.index = index;
        this.originalText = originalText;
        this.text = text;
        this.position = position;
        this.columnI = columnI;
        this.isToken = isToken;
        this.kind = kind;
      }
  
      @Override
      public int getIndex() {
        return index;
      }
  
      @Override
      public String getText() {
        return text;
      }
  
      @Override
      public String getOriginalText() {
        return originalText;
      }
  
      @Override
      public int length() {
        return originalText.length();
      }
  
      @Override
      public int getPosition() {
        return position;
      }
  
      @Override
      public int getColumn() {
        return columnI;
      }
  
      boolean isToken() {
        return isToken;
      }
  
      @Override
      public boolean isNewline() {
        return Newlines.isNewline(text);
      }
  
      @Override
      public boolean isSlashSlashComment() {
        return text.startsWith("//");
      }
  
      @Override
      public boolean isSlashStarComment() {
        return text.startsWith("/*");
      }
  
      @Override
      public boolean isJavadocComment() {
        // comments like `/***` are also javadoc, but their formatting probably won't be improved
        // by the javadoc formatter
        return text.startsWith("/**") && text.charAt("/**".length()) != '*' && text.length() > 4;
      }
  
      @Override
      public boolean isComment() {
        return isSlashSlashComment() || isSlashStarComment();
      }
  
      @Override
      public String toString() {
        return MoreObjects.toStringHelper(this)
            .add("index", index)
            .add("text", text)
            .add("position", position)
            .add("columnI", columnI)
            .add("isToken", isToken)
            .toString();
      }
  
      public TokenKind kind() {
        return kind;
      }
    }
  
    /**
     * A {@link Token} contains a token {@link Tok} and its associated non-tokens; each non-token
     * {@link Tok} belongs to one {@link Token}. Each {@link Token} has an immutable list of its
     * non-tokens that appear before it, and another list of its non-tokens that appear after it. The
     * concatenation of the texts of all the {@link Token}s' {@link Tok}s, each preceded by the texts
     * of its {@code toksBefore} and followed by the texts of its {@code toksAfter}, equals the input.
     */
    static final class Token implements Input.Token {
      private final Tok tok;
      private final ImmutableList<Tok> toksBefore;
      private final ImmutableList<Tok> toksAfter;
  
      /**
       * Token constructor.
       *
       * @param toksBefore the earlier non-token {link Tok}s assigned to this {@code Token}
       * @param tok this token {@link Tok}
       * @param toksAfter the later non-token {link Tok}s assigned to this {@code Token}
       */
      Token(List<Tok> toksBefore, Tok tok, List<Tok> toksAfter) {
        this.toksBefore = ImmutableList.copyOf(toksBefore);
        this.tok = tok;
        this.toksAfter = ImmutableList.copyOf(toksAfter);
      }
  
      /**
       * Get the token's {@link Tok}.
       *
       * @return the token's {@link Tok}
       */
      @Override
      public Tok getTok() {
        return tok;
      }
  
      /**
       * Get the earlier {@link Tok}s assigned to this {@code Token}.
       *
       * @return the earlier {@link Tok}s assigned to this {@code Token}
       */
      @Override
      public ImmutableList<? extends Input.Tok> getToksBefore() {
        return toksBefore;
      }
  
      /**
       * Get the later {@link Tok}s assigned to this {@code Token}.
       *
       * @return the later {@link Tok}s assigned to this {@code Token}
       */
      @Override
      public ImmutableList<? extends Input.Tok> getToksAfter() {
        return toksAfter;
      }
  
      @Override
      public String toString() {
        return MoreObjects.toStringHelper(this)
            .add("tok", tok)
            .add("toksBefore", toksBefore)
            .add("toksAfter", toksAfter)
            .toString();
      }
    }
  
    private final String text; // The input.
    private int kN; // The number of numbered toks (tokens or comments), excluding the EOF.
  
    /*
     * The following lists record the sequential indices of the {@code Tok}s on each input line. (Only
     * tokens and comments have sequential indices.) Tokens and {@code //} comments lie on just one
     * line; {@code /*} comments can lie on multiple lines. These data structures (along with
     * equivalent ones for the formatted output) let us compute correspondences between the input and
     * output.
     */
  
    private final ImmutableMap<Integer, Integer> positionToColumnMap; // Map Tok position to column.
    private final ImmutableList<Token> tokens; // The Tokens for this input.
    private final ImmutableRangeMap<Integer, Token> positionTokenMap; // Map position to Token.
  
    /** Map from Tok index to the associated Token. */
    private final Token[] kToToken;
  
    /**
     * Input constructor.
     *
     * @param text the input text
     * @throws FormatterException if the input cannot be parsed
     */
    public JavaInput(String text) throws FormatterException {
      this.text = checkNotNull(text);
      setLines(ImmutableList.copyOf(Newlines.lineIterator(text)));
      ImmutableList<Tok> toks = buildToks(text);
      positionToColumnMap = makePositionToColumnMap(toks);
      tokens = buildTokens(toks);
      ImmutableRangeMap.Builder<Integer, Token> tokenLocations = ImmutableRangeMap.builder();
      for (Token token : tokens) {
        Input.Tok end = JavaOutput.endTok(token);
        int upper = end.getPosition();
        if (!end.getText().isEmpty()) {
          upper += end.length() - 1;
        }
        tokenLocations.put(Range.closed(JavaOutput.startTok(token).getPosition(), upper), token);
      }
      positionTokenMap = tokenLocations.build();
  
      // adjust kN for EOF
      kToToken = new Token[kN + 1];
      for (Token token : tokens) {
        for (Input.Tok tok : token.getToksBefore()) {
          if (tok.getIndex() < 0) {
            continue;
          }
          kToToken[tok.getIndex()] = token;
        }
        kToToken[token.getTok().getIndex()] = token;
        for (Input.Tok tok : token.getToksAfter()) {
          if (tok.getIndex() < 0) {
            continue;
          }
          kToToken[tok.getIndex()] = token;
        }
      }
    }
  
    private static ImmutableMap<Integer, Integer> makePositionToColumnMap(List<Tok> toks) {
      ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builder();
      for (Tok tok : toks) {
        builder.put(tok.getPosition(), tok.getColumn());
      }
      return builder.buildOrThrow();
    }
  
    /**
     * Get the input text.
     *
     * @return the input text
     */
    @Override
    public String getText() {
      return text;
    }
  
    @Override
    public ImmutableMap<Integer, Integer> getPositionToColumnMap() {
      return positionToColumnMap;
    }
  
    /** Lex the input and build the list of toks. */
    private ImmutableList<Tok> buildToks(String text) throws FormatterException {
      ImmutableList<Tok> toks = buildToks(text, ImmutableSet.of());
      kN = getLast(toks).getIndex();
      computeRanges(toks);
      return toks;
    }
  
    /**
     * Lex the input and build the list of toks.
     *
     * @param text the text to be lexed.
     * @param stopTokens a set of tokens which should cause lexing to stop. If one of these is found,
     *     the returned list will include tokens up to but not including that token.
     */
    static ImmutableList<Tok> buildToks(String text, ImmutableSet<TokenKind> stopTokens)
        throws FormatterException {
      stopTokens = ImmutableSet.<TokenKind>builder().addAll(stopTokens).add(TokenKind.EOF).build();
      Context context = new Context();
      Options.instance(context).put("--enable-preview", "true");
      JavaFileManager fileManager = new JavacFileManager(context, false, UTF_8);
      context.put(JavaFileManager.class, fileManager);
      DiagnosticCollector<JavaFileObject> diagnosticCollector = new DiagnosticCollector<>();
      context.put(DiagnosticListener.class, diagnosticCollector);
      Log log = Log.instance(context);
      log.useSource(
          new SimpleJavaFileObject(URI.create("Source.java"), Kind.SOURCE) {
            @Override
            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
              return text;
            }
          });
      DeferredDiagnosticHandler diagnostics = new DeferredDiagnosticHandler(log);
      ImmutableList<RawTok> rawToks = JavacTokens.getTokens(text, context, stopTokens);
      if (diagnostics.getDiagnostics().stream().anyMatch(d -> d.getKind() == Diagnostic.Kind.ERROR)) {
        return ImmutableList.of(new Tok(0, "", "", 0, 0, true, null)); // EOF
      }
      int kN = 0;
      List<Tok> toks = new ArrayList<>();
      int charI = 0;
      int columnI = 0;
      for (RawTok t : rawToks) {
        if (stopTokens.contains(t.kind())) {
          break;
        }
        int charI0 = t.pos();
        // Get string, possibly with Unicode escapes.
        String originalTokText = text.substring(charI0, t.endPos());
        String tokText =
            t.kind() == TokenKind.STRINGLITERAL
                ? t.stringVal() // Unicode escapes removed.
                : originalTokText;
        char tokText0 = tokText.charAt(0); // The token's first character.
        final boolean isToken; // Is this tok a token?
        final boolean isNumbered; // Is this tok numbered? (tokens and comments)
        String extraNewline = null; // Extra newline at end?
        List<String> strings = new ArrayList<>();
        if (Character.isWhitespace(tokText0)) {
          isToken = false;
          isNumbered = false;
          Iterator<String> it = Newlines.lineIterator(originalTokText);
          while (it.hasNext()) {
            String line = it.next();
            String newline = Newlines.getLineEnding(line);
            if (newline != null) {
              String spaces = line.substring(0, line.length() - newline.length());
              if (!spaces.isEmpty()) {
                strings.add(spaces);
              }
              strings.add(newline);
            } else if (!line.isEmpty()) {
              strings.add(line);
            }
          }
        } else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
          isToken = true;
          isNumbered = true;
          strings.add(originalTokText);
        } else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
          // For compatibility with an earlier lexer, the newline after a // comment is its own tok.
          if (tokText.startsWith("//")
              && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) {
            extraNewline = Newlines.getLineEnding(originalTokText);
            tokText = tokText.substring(0, tokText.length() - extraNewline.length());
            originalTokText =
                originalTokText.substring(0, originalTokText.length() - extraNewline.length());
          }
          isToken = false;
          isNumbered = true;
          strings.add(originalTokText);
        } else if (Character.isJavaIdentifierStart(tokText0)
            || Character.isDigit(tokText0)
            || (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) {
          // Identifier, keyword, or numeric literal (a dot may begin a number, as in .2D).
          isToken = true;
          isNumbered = true;
          strings.add(tokText);
        } else {
          // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
          // cannot be lexed without syntactic knowledge. This implementation fails if the token
          // contains Unicode escapes.
          isToken = true;
          isNumbered = true;
          for (char c : tokText.toCharArray()) {
            strings.add(String.valueOf(c));
          }
        }
        if (strings.size() == 1) {
          toks.add(
              new Tok(
                  isNumbered ? kN++ : -1,
                  originalTokText,
                  tokText,
                  charI,
                  columnI,
                  isToken,
                  t.kind()));
          charI += originalTokText.length();
          columnI = updateColumn(columnI, originalTokText);
  
        } else {
          if (strings.size() != 1 && !tokText.equals(originalTokText)) {
            throw new FormatterException(
                "Unicode escapes not allowed in whitespace or multi-character operators");
          }
          for (String str : strings) {
            toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null));
            charI += str.length();
            columnI = updateColumn(columnI, originalTokText);
          }
        }
        if (extraNewline != null) {
          toks.add(new Tok(-1, extraNewline, extraNewline, charI, columnI, false, null));
          columnI = 0;
          charI += extraNewline.length();
        }
      }
      toks.add(new Tok(kN, "", "", charI, columnI, true, null)); // EOF tok.
      return ImmutableList.copyOf(toks);
    }
  
    private static int updateColumn(int columnI, String originalTokText) {
      Integer last = Iterators.getLast(Newlines.lineOffsetIterator(originalTokText));
      if (last > 0) {
        columnI = originalTokText.length() - last;
      } else {
        columnI += originalTokText.length();
      }
      return columnI;
    }
  
    private static ImmutableList<Token> buildTokens(List<Tok> toks) {
      ImmutableList.Builder<Token> tokens = ImmutableList.builder();
      int k = 0;
      int kN = toks.size();
  
      // Remaining non-tokens before the token go here.
      ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder();
  
      OUTERMOST:
      while (k < kN) {
        while (!toks.get(k).isToken()) {
          Tok tok = toks.get(k++);
          toksBefore.add(tok);
          if (isParamComment(tok)) {
            while (toks.get(k).isNewline()) {
              // drop newlines after parameter comments
              k++;
            }
          }
        }
        Tok tok = toks.get(k++);
  
        // Non-tokens starting on the same line go here too.
        ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder();
        OUTER:
        while (k < kN && !toks.get(k).isToken()) {
          // Don't attach inline comments to certain leading tokens, e.g. for `f(/*flag1=*/true).
          //
          // Attaching inline comments to the right token is hard, and this barely
          // scratches the surface. But it's enough to do a better job with parameter
          // name comments.
          //
          // TODO(cushon): find a better strategy.
          if (toks.get(k).isSlashStarComment()) {
            switch (tok.getText()) {
              case "(":
              case "<":
              case ".":
                break OUTER;
              default:
                break;
            }
          }
          if (toks.get(k).isJavadocComment()) {
            switch (tok.getText()) {
              case ";":
                break OUTER;
              default:
                break;
            }
          }
          if (isParamComment(toks.get(k))) {
            tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
            toksBefore = ImmutableList.<Tok>builder().add(toks.get(k++));
            // drop newlines after parameter comments
            while (toks.get(k).isNewline()) {
              k++;
            }
            continue OUTERMOST;
          }
          Tok nonTokenAfter = toks.get(k++);
          toksAfter.add(nonTokenAfter);
          if (Newlines.containsBreaks(nonTokenAfter.getText())) {
            break;
          }
        }
        tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
        toksBefore = ImmutableList.builder();
      }
      return tokens.build();
    }
  
    private static boolean isParamComment(Tok tok) {
      return tok.isSlashStarComment()
          && tok.getText().matches("\\/\\*[A-Za-z0-9\\s_\\-]+=\\s*\\*\\/");
    }
  
    /**
     * Convert from a character range to a token range.
     *
     * @param characterRange the {@code 0}-based {@link Range} of characters
     * @return the {@code 0}-based {@link Range} of tokens
     * @throws FormatterException if the upper endpoint of the range is outside the file
     */
    Range<Integer> characterRangeToTokenRange(Range<Integer> characterRange)
        throws FormatterException {
      if (characterRange.upperEndpoint() > text.length()) {
        throw new FormatterException(
            String.format(
                "error: invalid length %d, offset + length (%d) is outside the file",
                characterRange.upperEndpoint() - characterRange.lowerEndpoint(),
                characterRange.upperEndpoint()));
      }
      // empty range stands for "format the line under the cursor"
      Range<Integer> nonEmptyRange =
          characterRange.isEmpty()
              ? Range.closedOpen(characterRange.lowerEndpoint(), characterRange.lowerEndpoint() + 1)
              : characterRange;
      ImmutableCollection<Token> enclosed =
          getPositionTokenMap().subRangeMap(nonEmptyRange).asMapOfRanges().values();
      if (enclosed.isEmpty()) {
        return EMPTY_RANGE;
      }
      return Range.closedOpen(
          enclosed.iterator().next().getTok().getIndex(), getLast(enclosed).getTok().getIndex() + 1);
    }
  
    /**
     * Get the number of toks.
     *
     * @return the number of toks, excluding the EOF tok
     */
    @Override
    public int getkN() {
      return kN;
    }
  
    /**
     * Get the Token by index.
     *
     * @param k the Tok index
     */
    @Override
    public Token getToken(int k) {
      return kToToken[k];
    }
  
    /**
     * Get the input tokens.
     *
     * @return the input tokens
     */
    @Override
    public ImmutableList<? extends Input.Token> getTokens() {
      return tokens;
    }
  
    /**
     * Get the navigable map from position to {@link Token}. Used to look for tokens following a given
     * one, and to implement the --offset and --length flags to reformat a character range in the
     * input file.
     *
     * @return the navigable map from position to {@link Token}
     */
    @Override
    public ImmutableRangeMap<Integer, Token> getPositionTokenMap() {
      return positionTokenMap;
    }
  
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this)
          .add("tokens", tokens)
          .add("super", super.toString())
          .toString();
    }
  
    private JCCompilationUnit unit;
  
    @Override
    public int getLineNumber(int inputPosition) {
      Verify.verifyNotNull(unit, "Expected compilation unit to be set.");
      return unit.getLineMap().getLineNumber(inputPosition);
    }
  
    @Override
    public int getColumnNumber(int inputPosition) {
      Verify.verifyNotNull(unit, "Expected compilation unit to be set.");
      return unit.getLineMap().getColumnNumber(inputPosition);
    }
  
    // TODO(cushon): refactor JavaInput so the CompilationUnit can be passed into
    // the constructor.
    public void setCompilationUnit(JCCompilationUnit unit) {
      this.unit = unit;
    }
  
    public RangeSet<Integer> characterRangesToTokenRanges(Collection<Range<Integer>> characterRanges)
        throws FormatterException {
      RangeSet<Integer> tokenRangeSet = TreeRangeSet.create();
      for (Range<Integer> characterRange : characterRanges) {
        tokenRangeSet.add(
            characterRangeToTokenRange(characterRange.canonical(DiscreteDomain.integers())));
      }
      return tokenRangeSet;
    }
  }
  