content: /*
   * Copyright 1999-2017 Alibaba Group Holding Ltd.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package com.alibaba.druid.sql.dialect.hive.parser;
  
  import com.alibaba.druid.DbType;
  import com.alibaba.druid.sql.parser.DialectFeature;
  import com.alibaba.druid.sql.parser.Keywords;
  import com.alibaba.druid.sql.parser.Lexer;
  import com.alibaba.druid.sql.parser.NotAllowCommentException;
  import com.alibaba.druid.sql.parser.ParserException;
  import com.alibaba.druid.sql.parser.SQLParserFeature;
  import com.alibaba.druid.sql.parser.Token;
  
  import java.util.HashMap;
  import java.util.Map;
  
  import static com.alibaba.druid.sql.parser.CharTypes.isWhitespace;
  import static com.alibaba.druid.sql.parser.DialectFeature.LexerFeature.*;
  import static com.alibaba.druid.sql.parser.DialectFeature.ParserFeature.*;
  import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI;
  import static com.alibaba.druid.sql.parser.Token.LITERAL_CHARS;
  
  public class HiveLexer extends Lexer {
      public static final Keywords HIVE_KEYWORDS;
      public static final DialectFeature HIVE_FEATURE = new DialectFeature();
      static {
          Map<String, Token> map = new HashMap<>();
  
          map.putAll(Keywords.DEFAULT_KEYWORDS.getKeywords());
  
          map.put("OF", Token.OF);
          map.put("CONCAT", Token.CONCAT);
          map.put("CONTINUE", Token.CONTINUE);
          map.put("MERGE", Token.MERGE);
          map.put("MATCHED", Token.MATCHED);
          map.put("USING", Token.USING);
  
          map.put("ROW", Token.ROW);
          map.put("LIMIT", Token.LIMIT);
          map.put("PARTITIONED", Token.PARTITIONED);
          map.put("PARTITION", Token.PARTITION);
          map.put("OVERWRITE", Token.OVERWRITE);
          //        map.put("SORT", Token.SORT);
          map.put("IF", Token.IF);
          map.put("TRUE", Token.TRUE);
          map.put("FALSE", Token.FALSE);
          map.put("RLIKE", Token.RLIKE);
          map.put("CONSTRAINT", Token.CONSTRAINT);
          map.put("DIV", Token.DIV);
          map.put("QUALIFY", Token.QUALIFY);
  
          HIVE_KEYWORDS = new Keywords(map);
          HIVE_FEATURE.configFeature(
                  ScanSQLTypeWithFrom,
                  NextTokenColon,
                  ScanAliasU,
                  JoinRightTableFrom,
                  GroupByAll,
                  SQLDateExpr,
                  ParseAssignItemRparenCommaSetReturn,
                  TableAliasLock,
                  TableAliasPartition,
                  AsSkip,
                  AsSequence,
                  AsDatabase,
                  AsDefault
          );
          HIVE_FEATURE.unconfigFeature(
                  PrimaryBangBangSupport
          );
      }
  
      @Override
      protected Keywords loadKeywords() {
          return HIVE_KEYWORDS;
      }
  
      public HiveLexer(String input, DbType dbType) {
          super(input, dbType);
          this.skipComment = true;
          this.keepComments = true;
          this.features |= SQLParserFeature.SupportUnicodeCodePoint.mask;
      }
  
      public HiveLexer(String input) {
          this(input, DbType.hive);
      }
  
      public HiveLexer(String input, SQLParserFeature... features) {
          this(input, DbType.hive, features);
      }
  
      public HiveLexer(String input, DbType dbType, SQLParserFeature... features) {
          super(input, dbType);
          this.skipComment = true;
          this.keepComments = true;
          this.features |= SQLParserFeature.SupportUnicodeCodePoint.mask;
          for (SQLParserFeature feature : features) {
              config(feature, true);
          }
      }
  
      @Override
      protected void scanString() {
          {
              boolean hasSpecial = false;
              int startIndex = pos + 1;
              int endIndex = -1; // text.indexOf('\'', startIndex);
              for (int i = startIndex; i < text.length(); ++i) {
                  final char ch = text.charAt(i);
                  if (ch == '\\') {
                      hasSpecial = true;
                      continue;
                  }
                  if (ch == '\'') {
                      endIndex = i;
                      break;
                  }
              }
  
              if (endIndex == -1) {
                  throw new ParserException("unclosed str. " + info());
              }
  
              String stringVal;
              if (token == Token.AS) {
                  stringVal = text.substring(pos, endIndex + 1);
              } else {
                  if (startIndex == endIndex) {
                      stringVal = "";
                  } else {
                      stringVal = text.substring(startIndex, endIndex);
                  }
              }
              // hasSpecial = stringVal.indexOf('\\') != -1;
  
              if (!hasSpecial) {
                  this.stringVal = stringVal;
                  int pos = endIndex + 1;
                  char ch = charAt(pos);
                  if (ch != '\'') {
                      this.pos = pos;
                      this.ch = ch;
                      token = LITERAL_CHARS;
                      return;
                  }
              }
          }
  
          mark = pos;
          boolean hasSpecial = false;
          for (; ; ) {
              if (isEOF()) {
                  lexError("unclosed.str.lit");
                  return;
              }
  
              ch = charAt(++pos);
  
              if (ch == '\\') {
                  scanChar();
                  if (!hasSpecial) {
                      initBuff(bufPos);
                      arraycopy(mark + 1, buf, 0, bufPos);
                      hasSpecial = true;
                  }
  
                  switch (ch) {
                      case '0':
                          putChar('\0');
                          break;
                      case '\'':
                          putChar('\'');
                          break;
                      case '"':
                          putChar('"');
                          break;
                      case 'b':
                          putChar('\b');
                          break;
                      case 'n':
                          putChar('\n');
                          break;
                      case 'r':
                          putChar('\r');
                          break;
                      case 't':
                          putChar('\t');
                          break;
                      case '\\':
                          putChar('\\');
                          break;
                      case 'Z':
                          putChar((char) 0x1A); // ctrl + Z
                          break;
                      case '%':
                          putChar('%');
                          break;
                      case '_':
                          putChar('_');
                          break;
                      case 'u':
                          if ((features & SQLParserFeature.SupportUnicodeCodePoint.mask) != 0) {
                              int codePointSize = 0;
                              for (int i = 0; i < 4; i++, codePointSize++) {
                                  char c = charAt(pos + 1 + i);
                                  if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) {
                                      break;
                                  }
                              }
  
                              int intVal = Integer.parseInt(text.substring(pos + 1, pos + 1 + codePointSize), 16);
                              pos += codePointSize;
  
                              putChar((char) intVal);
                          } else {
                              putChar(ch);
                          }
                          break;
                      default:
                          putChar(ch);
                          break;
                  }
  
                  continue;
              }
              if (ch == '\'') {
                  scanChar();
                  if (ch != '\'') {
                      token = LITERAL_CHARS;
                      break;
                  } else {
                      if (!hasSpecial) {
                          initBuff(bufPos);
                          arraycopy(mark + 1, buf, 0, bufPos);
                          hasSpecial = true;
                      }
                      putChar('\'');
                      continue;
                  }
              }
  
              if (!hasSpecial) {
                  bufPos++;
                  continue;
              }
  
              if (bufPos == buf.length) {
                  putChar(ch);
              } else {
                  buf[bufPos++] = ch;
              }
          }
  
          if (!hasSpecial) {
              stringVal = subString(mark + 1, bufPos);
          } else {
              stringVal = new String(buf, 0, bufPos);
          }
      }
  
      public void scanComment() {
          if (ch != '/' && ch != '-') {
              throw new IllegalStateException();
          }
  
          Token lastToken = this.token;
  
          mark = pos;
          bufPos = 0;
          scanChar();
  
          if (ch == ' ') {
              mark = pos;
              bufPos = 0;
              scanChar();
  
              if (dialectFeatureEnabled(ScanHiveCommentDoubleSpace) && ch == ' ') {
                  mark = pos;
                  bufPos = 0;
                  scanChar();
              }
          }
  
          // /*+ */
          if (ch == '*') {
              scanChar();
              bufPos++;
  
              while (ch == ' ') {
                  scanChar();
                  bufPos++;
              }
  
              boolean isHint = false;
              int startHintSp = bufPos + 1;
              if (ch == '+') {
                  isHint = true;
                  scanChar();
                  bufPos++;
              }
  
              for (; ; ) {
                  if (ch == '*') {
                      if (charAt(pos + 1) == '/') {
                          bufPos += 2;
                          scanChar();
                          scanChar();
                          break;
                      } else if (isWhitespace(charAt(pos + 1))) {
                          int i = 2;
                          for (; i < 1024 * 1024; ++i) {
                              if (!isWhitespace(charAt(pos + i))) {
                                  break;
                              }
                          }
                          if (charAt(pos + i) == '/') {
                              bufPos += 2;
                              pos += (i + 1);
                              ch = charAt(pos);
                              break;
                          }
                      }
                  }
  
                  scanChar();
                  if (ch == EOI) {
                      break;
                  }
                  bufPos++;
              }
  
              if (isHint) {
                  stringVal = subString(mark + startHintSp, (bufPos - startHintSp) - 1);
                  token = Token.HINT;
              } else {
                  stringVal = subString(mark, bufPos + 1);
                  token = Token.MULTI_LINE_COMMENT;
                  commentCount++;
                  if (keepComments) {
                      addComment(stringVal);
                  }
              }
  
              if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
                  return;
              }
  
              if (token != Token.HINT && !isAllowComment()) {
                  throw new NotAllowCommentException();
              }
  
              return;
          }
  
          if (!isAllowComment()) {
              throw new NotAllowCommentException();
          }
  
          if (ch == '/' || ch == '-') {
              scanChar();
              bufPos++;
  
              for (; ; ) {
                  if (ch == '\r') {
                      if (charAt(pos + 1) == '\n') {
                          line++;
                          bufPos += 2;
                          scanChar();
                          break;
                      }
                      bufPos++;
                      break;
                  } else if (ch == EOI) {
                      if (pos >= text.length()) {
                          break;
                      }
                  }
  
                  if (ch == '\n') {
                      line++;
                      scanChar();
                      bufPos++;
                      break;
                  }
  
                  scanChar();
                  bufPos++;
              }
  
              stringVal = subString(mark, ch != EOI ? bufPos : bufPos + 1);
              token = Token.LINE_COMMENT;
              commentCount++;
              if (keepComments) {
                  addComment(stringVal);
              }
              endOfComment = isEOF();
  
              if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
                  return;
              }
  
              return;
          }
      }
  
      @Override
      protected void initDialectFeature() {
          this.dialectFeature = HIVE_FEATURE;
      }
  }
  