content: -- Copyright (c) 2017 Uber Technologies, Inc.
  --
  -- Permission is hereby granted, free of charge, to any person obtaining a copy
  -- of this software and associated documentation files (the "Software"), to deal
  -- in the Software without restriction, including without limitation the rights
  -- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  -- copies of the Software, and to permit persons to whom the Software is
  -- furnished to do so, subject to the following conditions:
  --
  -- The above copyright notice and this permission notice shall be included in
  -- all copies or substantial portions of the Software.
  --
  -- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  -- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  -- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  -- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  -- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  -- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  -- THE SOFTWARE.
  
  {-# LANGUAGE DeriveFunctor #-}
  {-# LANGUAGE DeriveTraversable #-}
  {-# LANGUAGE StandaloneDeriving #-}
  {-# LANGUAGE FlexibleContexts #-}
  {-# LANGUAGE TypeFamilies #-}
  {-# LANGUAGE UndecidableInstances #-}
  
  module Database.Sql.Hive.Parser where
  
  import Database.Sql.Type
  import Database.Sql.Info
  import Database.Sql.Helpers
  import Database.Sql.Hive.Type as HT
  
  import Database.Sql.Hive.Scanner
  import Database.Sql.Hive.Parser.Internal
  import Database.Sql.Position
  
  import qualified Database.Sql.Hive.Parser.Token as Tok
  
  import           Control.Monad (void)
  import           Control.Monad.Reader (runReader, local, asks)
  import           Data.Char (isDigit)
  import           Data.Text.Lazy (Text)
  import qualified Data.Text.Lazy as TL
  import qualified Data.Text.Lazy.Encoding as TL
  import           Data.Set (Set)
  import qualified Data.Set as S
  import qualified Data.List as L
  
  import Data.Maybe (fromMaybe)
  import Data.Monoid (Endo (..))
  import qualified Text.Parsec as P
  import           Text.Parsec ( chainl1, choice, many
                               , option, optional, optionMaybe
                               , sepBy, sepBy1, try, (<|>), (<?>))
  
  
  import Data.Semigroup (Semigroup (..), sconcat)
  import Data.List.NonEmpty (NonEmpty ((:|)))
  import Data.Foldable (fold)
  
  statementParser :: Parser (HiveStatement RawNames Range)
  statementParser = do
      maybeStmt <- optionMaybe $ choice
          [ HiveUseStmt <$> useP
          , HiveAnalyzeStmt <$> analyzeP
          , do
                let options =
                      -- this list is hive-specific statement types that may be
                      -- preceded by an optional `WITH` and an optional inverted
                      -- `FROM`
                      [ (void insertDirectoryPrefixP, fmap HiveInsertDirectoryStmt . insertDirectoryP)
                      ]
                    prefixes = map fst options
                    baseParsers = map snd options
                _ <- try $ P.lookAhead $ optional withP >> invertedFromP >> choice prefixes
                with <- option id withP
                invertedFrom <- invertedFromP
                let parsers = map ($ (with, invertedFrom)) baseParsers
                choice $ parsers
          , try $ HiveTruncatePartitionStmt <$> truncatePartitionStatementP
          , HiveUnhandledStatement <$> describeP
          , HiveUnhandledStatement <$> showP
          , do
                _ <- try $ P.lookAhead createFunctionPrefixP
                HiveUnhandledStatement <$> createFunctionP
          , do
                _ <- try $ P.lookAhead dropFunctionPrefixP
                HiveUnhandledStatement <$> dropFunctionP
          , HiveStandardSqlStatement <$> statementP
          , try $ HiveAlterTableSetLocationStmt <$> alterTableSetLocationP
          , try $ HiveUnhandledStatement <$> alterTableSetTblPropertiesP
          , alterPartitionP
          , HiveSetPropertyStmt <$> setP
          , HiveUnhandledStatement <$> reloadFunctionP
          ]
      case maybeStmt of
          Just stmt -> terminator >> return stmt
          Nothing -> HiveStandardSqlStatement <$> emptyStatementP
    where
      terminator = (Tok.semicolonP <|> eof) -- normal statements may be terminated by `;` or eof
      emptyStatementP = EmptyStmt <$> Tok.semicolonP  -- but we don't allow eof here. `;` is the
      -- only way to write the empty statement, i.e. `` (empty string) is not allowed.
  
  
  emptyParserScope :: ParserScope
  emptyParserScope = ParserScope
      { selectTableAliases = Nothing }
  
  -- | parse consumes a statement, or fails
  parse :: Text -> Either P.ParseError (HiveStatement RawNames Range)
  parse text = runReader (P.runParserT statementParser 0 "-"  . tokenize $ text) emptyParserScope
  
  -- | parseAll consumes all input as a single statement, or fails
  parseAll :: Text -> Either P.ParseError (HiveStatement RawNames Range)
  parseAll text = runReader (P.runParserT (statementParser <* P.eof) 0 "-"  . tokenize $ text) emptyParserScope
  
  -- | parseMany consumes multiple statements, or fails
  parseMany :: Text -> Either P.ParseError [HiveStatement RawNames Range]
  parseMany text = runReader (P.runParserT (P.many1 statementParser) 0 "-"  . tokenize $ text) emptyParserScope
  
  -- | parseManyAll consumes all input multiple statements, or fails
  parseManyAll :: Text -> Either P.ParseError [HiveStatement RawNames Range]
  parseManyAll text = runReader (P.runParserT (P.many1 statementParser <* P.eof) 0 "-"  . tokenize $ text) emptyParserScope
  
  -- | parseManyEithers consumes all input as multiple (statements or failures)
  -- it should never fail
  parseManyEithers :: Text -> Either P.ParseError [Either (Unparsed Range) (HiveStatement RawNames Range)]
  parseManyEithers text = runReader (P.runParserT parser 0 "-"  . tokenize $ text) emptyParserScope
    where
      parser = do
          statements <- P.many1 $ P.setState 0 >> choice
              [ try $ Right <$> statementParser
              , try $ Left <$> do
                  ss  <- many Tok.notSemicolonP
                  e <- Tok.semicolonP
                  pure $ case ss of
                      [] -> Unparsed e
                      s:_ -> Unparsed (s <> e)
              ]
  
          locs <- many Tok.notSemicolonP
          P.eof
          pure $ case locs of
              [] -> statements
              s:es -> statements ++ [Left $ Unparsed $ sconcat (s:|es)]
  
  optionBool :: Parser a -> Parser Bool
  optionBool p = option False $ p >> pure True
  
  statementP :: Parser (Statement Hive RawNames Range)
  statementP = choice
      [ do
            let options =
                  -- this list is universal statement types that may be preceded
                  -- by an optional `WITH` and an optional inverted `FROM`
                  [ (void Tok.insertP, fmap InsertStmt . insertP)
                  , (void Tok.selectP, fmap QueryStmt . queryP )
                  ]
                prefixes = map fst options
                baseParsers = map snd options
            _ <- try $ P.lookAhead $ optional withP >> invertedFromP >> choice prefixes
            with <- option id withP
            invertedFrom <- invertedFromP
            let parsers = map ($ (with, invertedFrom)) baseParsers
            choice $ parsers
      , InsertStmt <$> loadDataInPathP
      , DeleteStmt <$> deleteP
      , explainP
      , TruncateStmt <$> truncateP
      , do
            _ <- try $ P.lookAhead createSchemaPrefixP
            CreateSchemaStmt <$> createSchemaP
      , do
            _ <- try $ P.lookAhead createViewPrefixP
            CreateViewStmt <$> createViewP
      , CreateTableStmt <$> createTableP
      , DropTableStmt <$> dropTableP
      , do
            _ <- try $ P.lookAhead alterTableRenameTablePrefixP
            AlterTableStmt <$> alterTableRenameTableP
      , do
            _ <- try $ P.lookAhead alterTableRenameColumnPrefixP
            AlterTableStmt <$> alterTableRenameColumnP
      , do
            _ <- try $ P.lookAhead alterTableAddColumnsPrefixP
            AlterTableStmt <$> alterTableAddColumnsP
      , GrantStmt <$> grantP
      , RevokeStmt <$> revokeP
      , CommitStmt <$> Tok.commitP
      , RollbackStmt <$> Tok.rollbackP
      ]
  
  useP :: Parser (Use Range)
  useP = do
    r <- Tok.useP
  
    use <- choice
      [ UseDefault <$> Tok.defaultP
      , UseDatabase . uncurry mkNormalSchema <$> Tok.schemaNameP
      ]
  
    return $ (r<>) <$> use
  
  
  analyzeP :: Parser (Analyze RawNames Range)
  analyzeP = do
      r <- Tok.analyzeP
      _ <- Tok.tableP
      tn <- tableNameP
      optional $ do
          _ <- Tok.partitionP
          partitionSpecP
  
      _ <- Tok.computeP
      e <- Tok.statisticsP
      e' <- consumeOrderedOptions e $
              [ do
                    _ <- Tok.forP
                    Tok.columnsP
              , do
                    _ <- Tok.cacheP
                    Tok.metadataP
              , Tok.noScanP
              ]
      return $ Analyze (r<>e') tn
  
  
  insertDirectoryPrefixP :: Parser (Range, InsertDirectoryLocale Range, Location Range)
  insertDirectoryPrefixP = do
      s <- Tok.insertP
      _ <- Tok.overwriteP
      insertDirectoryLocale <- insertDirectoryLocaleP
      insertDirectoryPath <- insertDirectoryPathP
  
      return (s, insertDirectoryLocale, insertDirectoryPath)
  
  insertDirectoryP :: (QueryPrefix, InvertedFrom) -> Parser (InsertDirectory RawNames Range)
  insertDirectoryP (with, farInvertedFrom) = do
      r <- Tok.insertP
      _ <- Tok.overwriteP
      insertDirectoryLocale <- insertDirectoryLocaleP
      insertDirectoryPath <- insertDirectoryPathP
  
      case farInvertedFrom of
          Just _ -> pure ()
          Nothing -> optional rowFormatP >> optional storedAsP
  
      insertDirectoryQuery <- case farInvertedFrom of
          Just _ -> querySelectP (with, farInvertedFrom)
          Nothing -> do
                         nearInvertedFrom <- invertedFromP
                         queryP (with, nearInvertedFrom)
      let insertDirectoryInfo = r <> (getInfo insertDirectoryQuery)
      return InsertDirectory{..}
    where
      rowFormatP :: Parser Range
      rowFormatP = do
          s <- Tok.rowP
          _ <- Tok.formatP
          e <- delimitedP
          return $ s <> e
  
  insertDirectoryLocaleP :: Parser (InsertDirectoryLocale Range)
  insertDirectoryLocaleP = do
      localToken <- optionMaybe Tok.localP
      let locale = case localToken of
              Just a -> InsertDirectoryLocal a
              Nothing -> InsertDirectoryHDFS
      return locale
  
  insertDirectoryPathP :: Parser (Location Range)
  insertDirectoryPathP = do
      r <- Tok.directoryP
      (path, r') <- Tok.stringP
      return $ HDFSPath (r <> r') path
  
  staticPartitionSpecItemP :: Parser (StaticPartitionSpecItem RawNames Range)
  staticPartitionSpecItemP = do
      col <- columnNameP
      _ <- Tok.equalP
      val <- constantP
      return $ StaticPartitionSpecItem (getInfo col <> getInfo val) col val
  
  staticPartitionSpecP :: Parser ([StaticPartitionSpecItem RawNames Range], Range)
  staticPartitionSpecP = do
      s <- Tok.openP
      items <- staticPartitionSpecItemP `sepBy1` Tok.commaP
      e <- Tok.closeP
      return (items, s <> e)
  
  type PartitionDecider = (Either
                              (StaticPartitionSpecItem RawNames Range)
                              (DynamicPartitionSpecItem RawNames Range))
  
  dynamicPartitionSpecItemP :: Parser (DynamicPartitionSpecItem RawNames Range)
  dynamicPartitionSpecItemP = do
      col <- columnNameP
      return $ DynamicPartitionSpecItem (getInfo col) col
  
  partitionSpecDeciderP :: Parser PartitionDecider
  partitionSpecDeciderP = do
      item <- choice
          [ do
                sp <- try $ staticPartitionSpecItemP
                return $ Left sp
          , do
                dp <- dynamicPartitionSpecItemP
                return $ Right dp
          ]
      return item
  
  partitionSpecP :: Parser ()
  partitionSpecP = do
      -- partitionSpecP currently does not tie down to any specific datatype.
      -- The datatype implementation is being deferred.
      _ <- Tok.openP
      items <- partitionSpecDeciderP `sepBy1` Tok.commaP
      _ <- Tok.closeP
      let dpSpec = L.foldl' specHelper dpSpecBase $ L.reverse items
      case dpSpec of
          Right _ -> return ()
          Left err -> fail err
    where
      specHelper :: (Either String ([StaticPartitionSpecItem RawNames Range],
                                    [DynamicPartitionSpecItem RawNames Range])) ->
                    PartitionDecider ->
                    (Either String ([StaticPartitionSpecItem RawNames Range],
                                    [DynamicPartitionSpecItem RawNames Range]))
      -- Note that specHelper reads partition cols from right to left
      -- This allows for list insertions to output a non-reversed list
      specHelper (Right (spItems, dpItems)) (Left spItem) =
          Right (spItem:spItems, dpItems)
  
      specHelper (Right (spItems, dpItems)) (Right dpItem) =
          case spItems of
              [] -> Right $ (spItems, dpItem:dpItems)
              _  -> Left  $ "Failed to parse partition \"" ++ show dpItem ++ "\": dynamic partition found preceding static partition"
  
      specHelper (Left s) _ = Left s
  
      dpSpecBase :: (Either String ([StaticPartitionSpecItem RawNames Range],
                                    [DynamicPartitionSpecItem RawNames Range]))
      dpSpecBase = Right ([], [])
  
  truncatePartitionStatementP :: Parser (TruncatePartition RawNames Range)
  truncatePartitionStatementP = do
      s <- Tok.truncateP
      _ <- Tok.tableP
      table <- tableNameP
      _ <- Tok.partitionP
      (_, e) <- staticPartitionSpecP
  
      let truncate' = Truncate (s <> getInfo table) table
  
      return $ TruncatePartition (s <> e) truncate'
  
  
  describeP :: Parser Range
  describeP = do
      s <- Tok.describeP
      e <- P.many1 Tok.notSemicolonP
      return $ s <> last e
  
  
  showP :: Parser Range
  showP = do
      s <- Tok.showP
      e <- P.many1 Tok.notSemicolonP
      return $ s <> last e
  
  
  createFunctionPrefixP :: Parser Range
  createFunctionPrefixP = do
      s <- Tok.createP
      optional Tok.temporaryP
      e <- Tok.functionP
      return $ s <> e
  
  createFunctionP :: Parser Range
  createFunctionP = do
     s <- createFunctionPrefixP
     e <- P.many1 Tok.notSemicolonP
     return $ s <> last e
  
  
  dropFunctionPrefixP :: Parser Range
  dropFunctionPrefixP = do
      s <- Tok.dropP
      optional Tok.temporaryP
      e <- Tok.functionP
      return $ s <> e
  
  dropFunctionP :: Parser Range
  dropFunctionP = do
     s <- dropFunctionPrefixP
     e <- P.many1 Tok.notSemicolonP
     return $ s <> last e
  
  alterTableSetLocationP :: Parser (AlterTableSetLocation RawNames Range)
  alterTableSetLocationP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      table <- tableNameP
      _ <- Tok.setP
      loc <- locationP
  
      let alterTableSetLocationInfo = s <> getInfo loc
          alterTableSetLocationTable = table
          alterTableSetLocationLocation = loc
  
      return AlterTableSetLocation{..}
  
  alterTableSetTblPropertiesP :: Parser Range
  alterTableSetTblPropertiesP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      _ <- tableNameP
      _ <- Tok.setP
      _ <- Tok.tblPropertiesP
      _ <- Tok.openP
      _ <- (Tok.stringP >> Tok.equalP >> Tok.stringP) `sepBy1` Tok.commaP
      e <- Tok.closeP
  
      return $ s <> e
  
  alterPartitionP :: Parser (HiveStatement RawNames Range)
  alterPartitionP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      tableName <- tableNameP
      choice $
          [ do
              _ <- Tok.partitionP
              (items, _) <- staticPartitionSpecP
              _ <- Tok.setP
              location <- locationP
              pure $ HiveAlterPartitionSetLocationStmt $ AlterPartitionSetLocation (s <> getInfo location) tableName items location
          , HiveUnhandledStatement . (s <>) <$> (addP <|> dropP)
          ]
  
    where
      addP :: Parser Range
      addP = do
          _ <- Tok.addP
          _ <- ifNotExistsP
  
          let partitionLocationP = do
                _ <- Tok.partitionP
                (_, e) <- staticPartitionSpecP
                option e (getInfo <$> locationP)
          last <$> P.many1 partitionLocationP
  
      dropP :: Parser Range
      dropP = do
          _ <- Tok.dropP
          optional $ ifExistsP
          (_, e) <- last <$> P.many1 (Tok.partitionP >> staticPartitionSpecP)
          consumeOrderedOptions e $
              [ Tok.ignoreP >> Tok.protectionP
              , Tok.purgeP
              ]
  
  
  setP :: Parser (SetProperty Range)
  setP = do
      s <- Tok.setP
      option (PrintProperties s "") $ choice $
        [ Tok.minusP >> Tok.keywordP "v" >> pure (PrintProperties s "-v")
        , do
          (name, pe)  <- Tok.propertyNameP
          choice $
            [ do
                _ <- Tok.equalP
                (setConfigValue, e) <- Tok.propertyValuePartP
                let details = SetPropertyDetails (s <> e) name setConfigValue
                pure (SetProperty details)
            , pure (PrintProperties (s <> pe) name)
            ]
        ]
  
  reloadFunctionP :: Parser Range
  reloadFunctionP = do
      s <- Tok.reloadP
      e <- Tok.functionP
      return $ s <> e
  
  
  insertBehaviorHelper :: InsertBehavior Range ->
                          Maybe (TablePartition) ->
                          InsertBehavior Range
  insertBehaviorHelper (InsertOverwrite a) (Just partition) = InsertOverwritePartition a partition
  insertBehaviorHelper (InsertAppend a) (Just partition) = InsertAppendPartition a partition
  insertBehaviorHelper ib _ = ib
  
  
  insertP :: (QueryPrefix, InvertedFrom) -> Parser (Insert RawNames Range)
  insertP (with, farInvertedFrom) = do
      r <- Tok.insertP
      insertBehaviorTok <- choice
          -- T407432 Overhaul Overwrite with Partition support
          [ do
              overwrite <- Tok.overwriteP
              return $ InsertOverwrite overwrite
          , do
              into <- Tok.intoP
              return $ InsertAppend into
          ]
  
      optional Tok.tableP
      insertTable <- tableNameP
  
      tablePartition <- optionMaybe $ do
          _ <- Tok.partitionP
          partitionSpecP
  
      let insertBehavior = insertBehaviorHelper insertBehaviorTok tablePartition
  
      insertColumns <- optionMaybe $ try $ do
          _ <- Tok.openP
          let oqColumnNameP = (\ (c, r') -> QColumnName r' Nothing c) <$> Tok.columnNameP
          c:cs <- oqColumnNameP `sepBy1` Tok.commaP
          _ <- Tok.closeP
          pure (c :| cs)
  
      insertValues <- choice
          [ do
              s <- Tok.valuesP
              (e, rows) <- rowsOfValuesP
              pure $ InsertExprValues (s <> e) rows
          , do
              isv <- case farInvertedFrom of
                Just _ -> InsertSelectValues <$> querySelectP (with, farInvertedFrom)
                Nothing -> InsertSelectValues <$> queryP (with, noInversion) -- INSERT .. FROM .. SELECT is not permitted
              pure $ isv
          ]
  
      let insertInfo = r <> (getInfo insertValues)
  
      pure Insert{..}
    where
      valueP :: Parser (DefaultExpr RawNames Range)
      valueP = do
          value <- constantP
          let r = getInfo value
          pure $ ExprValue $ ConstantExpr r value
  
      rowOfValuesP = do
          s <- Tok.openP
          x:xs <- valueP `sepBy1` Tok.commaP
          e <- Tok.closeP
          pure $ (s <> e, x :| xs)
  
      rowsOfValuesP = do
          rows <- rowOfValuesP `sepBy1` Tok.commaP
          let infos = map fst rows
              r:rs = map snd rows
          pure $ (head infos <> last infos, r :| rs)
  
  
  loadDataInPathP :: Parser (Insert RawNames Range)
  loadDataInPathP = do
      s <- Tok.loadP
      _ <- Tok.dataP
      optional Tok.localP
      _ <- Tok.inPathP
      (path, r) <- Tok.stringP
      maybeOverwrite <- optionMaybe Tok.overwriteP
      into <- Tok.intoP
      _ <- Tok.tableP
      table <- tableNameP
      partitions <- optionMaybe $ do
          _ <- Tok.partitionP
          snd <$> staticPartitionSpecP
  
      let e = maybe (getInfo table) id partitions
          insertInfo = s <> e
          behaviorTok = case maybeOverwrite of
              Nothing -> InsertAppend into
              Just overwrite -> InsertOverwrite overwrite
          insertBehavior = insertBehaviorHelper behaviorTok (void partitions)
          insertTable = table
          insertColumns = Nothing
          insertValues = InsertDataFromFile r path
  
      pure Insert{..}
  
  deleteP :: Parser (Delete RawNames Range)
  deleteP = do
      r <- Tok.deleteP
  
      _ <- Tok.fromP
      table <- tableNameP
  
      maybeExpr <- optionMaybe $ do
          _ <- Tok.whereP
          local (introduceAliases $ tableNameToTableAlias table) exprP
  
      let r' = case maybeExpr of
            Nothing -> getInfo table
            Just expr -> getInfo expr
          info = r <> r'
  
      pure $ Delete info table maybeExpr
  
  
  truncateP :: Parser (Truncate RawNames Range)
  truncateP = do
      s <- Tok.truncateP
      _ <- Tok.tableP
      table <- tableNameP
  
      pure $ Truncate (s <> getInfo table) table
  
  
  type QueryPrefix = Query RawNames Range -> Query RawNames Range
  
  emptyPrefix :: QueryPrefix
  emptyPrefix = id
  
  withP :: Parser QueryPrefix
  withP = do
      r <- Tok.withP
      withs <- cteP `sepBy1` Tok.commaP
  
      return $ \ query ->
          let r' = sconcat $ r :| getInfo query : map cteInfo withs
           in QueryWith r' withs query
    where
      cteP = do
          alias <- tableAliasP
          columns <- option []
              $ P.between Tok.openP Tok.closeP $ columnAliasP `sepBy1` Tok.commaP
  
          _ <- Tok.asP
  
          (query, r') <- do
              _ <- Tok.openP
              invertedFrom <- invertedFromP
              q <- queryP (emptyPrefix, invertedFrom) -- a query may not have more than 1 WITH clause.
              r' <- Tok.closeP
              return (q, r')
  
          return $ CTE (getInfo alias <> r') alias columns query
  
  -- parses just a SELECT, consuming no UNIONs
  -- i.e. it returns only QuerySelects
  querySelectP :: (QueryPrefix, InvertedFrom) -> Parser (Query RawNames Range)
  querySelectP (with, invertedFrom) = queryPHelper with invertedFrom False
  
  -- parses SELECTs and UNIONs
  -- i.e. it returns QuerySelects and QueryUnions
  queryP :: (QueryPrefix, InvertedFrom) -> Parser (Query RawNames Range)
  queryP (with, invertedFrom) = queryPHelper with invertedFrom True
  
  queryPHelper :: QueryPrefix -> InvertedFrom -> Bool -> Parser (Query RawNames Range)
  queryPHelper with invertedFrom unionsPermitted = do
      -- The invertedFrom, if supplied, will only be applied to the first SELECT.
      -- If invertedFrom is Nothing, then it is assumed the first SELECT has no inversion.
      firstSelect <- onlySelectP invertedFrom
  
      query <- if unionsPermitted
                   then do
                       maybeUnion <- optionMaybe unionP
                       case maybeUnion of
                           Nothing -> return firstSelect
                           Just union -> do
                               let subsequentSelectP = do
                                       nextInvertedFrom <- invertedFromP
                                       onlySelectP nextInvertedFrom
                               subsequentSelects <- subsequentSelectP `chainl1` unionP
                               return $ union firstSelect subsequentSelects
                   else return firstSelect
  
      order <- option id orderP
      optional selectClusterP
      limit <- option id limitP
  
      return $ with $ limit $ order query
    where
      onlySelectP invertedFrom' = do
          select <- selectP invertedFrom'
          return $ QuerySelect (selectInfo select) select
  
      unionP = do
          r <- Tok.unionP
          distinct <- option (Distinct True) distinctP
          return $ QueryUnion r distinct Unused
  
      orderP =  do
          (r, orders) <- orderTopLevelP
          return $ \ query -> QueryOrder (getInfo query <> r) orders query
  
      limitP = do
          r <- Tok.limitP
          Tok.numberP >>= \ (v, r') ->
              let limit = Limit (r <> r') v
               in return $ \ query -> QueryLimit (getInfo query <> r') limit query
  
  
  distinctP :: Parser Distinct
  distinctP = choice $
      [ Tok.allP >> return (Distinct False)
      , Tok.distinctP >> return (Distinct True)
      ]
  
  
  explainP :: Parser (Statement Hive RawNames Range)
  explainP = do
      s <- Tok.explainP
      stmt <- choice
          [ InsertStmt <$> insertP (emptyPrefix, noInversion)
          , DeleteStmt <$> deleteP
          , QueryStmt <$> queryP (emptyPrefix, noInversion)
          ]
  
      pure $ ExplainStmt (s <> getInfo stmt) stmt
  
  
  tableAliasP :: Parser (TableAlias Range)
  tableAliasP = do
      (name, r) <- Tok.tableNameP
      makeTableAlias r name
  
  
  columnAliasP :: Parser (ColumnAlias Range)
  columnAliasP = do
      (name, r) <- Tok.columnNameP
      makeColumnAlias r name
  
  
  createSchemaPrefixP :: Parser Range
  createSchemaPrefixP = do
      s <- Tok.createP
      e <- Tok.schemaP <|> Tok.databaseP
      return $ s <> e
  
  
  ifNotExistsP :: Parser (Maybe Range)
  ifNotExistsP = optionMaybe $ do
      s' <- Tok.ifP
      _ <- Tok.notP
      e' <- Tok.existsP
      pure $ s' <> e'
  
  
  commentP :: Parser Range
  commentP = do
      s <- Tok.commentP
      (_, e) <- Tok.stringP
      return $ s <> e
  
  
  locationP :: Parser (Location Range)
  locationP = do
      s <- Tok.locationP
      (loc, e) <- Tok.stringP
      return $ HDFSPath (s <> e) loc
  
  
  createSchemaP :: Parser (CreateSchema RawNames Range)
  createSchemaP = do
      s <- createSchemaPrefixP
      createSchemaIfNotExists <- ifNotExistsP
  
      (name, r) <- Tok.schemaNameP
      let createSchemaName = mkNormalSchema name r
  
      e <- consumeOrderedOptions r $
              [ commentP
              , getInfo <$> locationP
              , dbPropertiesP
              ]
      let createSchemaInfo = s <> e
  
      return $ CreateSchema{..}
  
    where
      dbPropertiesP = do
          s <- Tok.withP
          _ <-Tok.dbPropertiesP
          _ <- Tok.openP
          _ <- propertyP `sepBy1` Tok.commaP
          e <- Tok.closeP
          return $ s <> e
  
  
  createViewPrefixP :: Parser Range
  createViewPrefixP = do
      s <- Tok.createP
      e <- Tok.viewP
      return $ s <> e
  
  createViewP :: Parser (CreateView RawNames Range)
  createViewP = do
      s <- createViewPrefixP
  
      let createViewPersistence = Persistent
      createViewIfNotExists <- ifNotExistsP
  
      createViewName <- tableNameP
  
      createViewColumns <- optionMaybe $ do
          _ <- Tok.openP
          c:cs <- flip sepBy1 Tok.commaP $ do
              col <- unqualifiedColumnNameP
              _ <- commentP
              return col
          _ <- Tok.closeP
          return (c:|cs)
  
      optional commentP
      optional $ do
          _ <- Tok.tblPropertiesP
          _ <- Tok.openP
          _ <- (Tok.stringP >> Tok.equalP >> Tok.stringP) `sepBy1` Tok.commaP
          Tok.closeP
  
      _ <- Tok.asP
      createViewQuery <- querySelectP (emptyPrefix, noInversion)
  
      let createViewInfo = s <> getInfo createViewQuery
      pure CreateView{..}
  
  
  data CreateTablePrefix r a = CreateTablePrefix
      { createTablePrefixInfo :: a
      , createTablePrefixPersistence :: Persistence a
      , createTablePrefixExternality :: Externality a
      , createTablePrefixIfNotExists :: Maybe a
      , createTablePrefixName :: CreateTableName r a
      }
  
  deriving instance ConstrainSNames Eq r a => Eq (CreateTablePrefix r a)
  deriving instance ConstrainSNames Show r a => Show (CreateTablePrefix r a)
  deriving instance ConstrainSASNames Functor r => Functor (CreateTablePrefix r)
  deriving instance ConstrainSASNames Foldable r => Foldable (CreateTablePrefix r)
  deriving instance ConstrainSASNames Traversable r => Traversable (CreateTablePrefix r)
  
  
  createTablePrefixP :: Parser (CreateTablePrefix RawNames Range)
  createTablePrefixP = do
      s <- Tok.createP
      createTablePrefixPersistence <- option Persistent $ Temporary <$> Tok.temporaryP
      createTablePrefixExternality <- option Internal (External <$> Tok.externalP)
      _ <- Tok.tableP
      createTablePrefixIfNotExists <- ifNotExistsP
      createTablePrefixName <- tableNameP
  
      let createTablePrefixInfo = s <> getInfo createTablePrefixName
  
      return CreateTablePrefix{..}
  
  
  createTableP :: Parser (CreateTable Hive RawNames Range)
  createTableP = choice
      [ do
            _ <- try $ P.lookAhead $ createTablePrefixP >> Tok.likeP
            createTableLikeP
      , createTableStandardP
      ]
  
  
  createTableLikeP :: Parser (CreateTable Hive RawNames Range)
  createTableLikeP = do
      CreateTablePrefix{..} <- createTablePrefixP
      let s = createTablePrefixInfo
          createTablePersistence = createTablePrefixPersistence
          createTableExternality = createTablePrefixExternality
          createTableIfNotExists = createTablePrefixIfNotExists
          createTableName = createTablePrefixName
  
      _ <- Tok.likeP
      table <- tableNameP
  
      let e = getInfo table
      e' <- option e $ choice $
          [ getInfo <$> locationP
          , storedAsP
          ]
  
      let createTableInfo = s <> e'
          createTableDefinition = TableLike (s <> e) table
          createTableExtra = Nothing
  
      return CreateTable{..}
  
  
  propertyP :: Parser (HiveMetadataProperty Range)
  propertyP = do
      (k, s) <- Tok.stringP
      _ <- Tok.equalP
      (v, e) <- Tok.stringP
      return $ HiveMetadataProperty (s <> e) k v
  
  storedAsP :: Parser Range
  storedAsP = do
      s <- Tok.storedP
      _ <- Tok.asP
      e <- choice
          [ Tok.orcP
          , Tok.sequenceFileP
          , Tok.textFileP
          , Tok.rcFileP
          , Tok.parquetP
          , Tok.avroP
          , do
                s' <- Tok.inputFormatP
                _ <- Tok.stringP
                _ <- Tok.outputFormatP
                (_, e') <- Tok.stringP
                return (s' <> e')
          ]
      return $ s <> e
  
  
  createTableStandardP :: Parser (CreateTable Hive RawNames Range)
  createTableStandardP = do
      CreateTablePrefix{..} <- createTablePrefixP
      let s = createTablePrefixInfo
          createTablePersistence = createTablePrefixPersistence
          createTableExternality = createTablePrefixExternality
          createTableIfNotExists = createTablePrefixIfNotExists
          createTableName = createTablePrefixName
  
      tableDefColumns <- optionMaybe createTableColumnsP
      let e1 = maybe s getInfo tableDefColumns
  
      e2 <- consumeOrderedOptions e1 $
              [ commentP
              , partitionedByP
              , clusteredByP
              , rowFormatP
              , storedAsP
              , getInfo <$> locationP
              ]
  
      tblProperties <- option Nothing (Just <$> tblPropertiesP)
  
      createTableDefinition <- case tableDefColumns of
          Just definition -> return definition
          Nothing -> choice
              [ createTableAsP
              , createTableNoColumnInfoP e2
              ]
  
      let e3 = getInfo createTableDefinition
          e4 = fromMaybe e2 (hiveMetadataPropertiesInfo <$> tblProperties)
          createTableExtra =
            Just HiveCreateTableExtra
            { hiveCreateTableExtraInfo = e3 <> e4
            , hiveCreateTableExtraTableProperties = tblProperties
            }
          createTableInfo = s <> e1 <> e2 <> e3 <> e4
  
      pure CreateTable{..}
  
    where
      columnDefinitionP = do
          (name, s) <- Tok.columnNameP
          columnDefinitionType <- dataTypeP
          optional commentP
  
          let columnDefinitionInfo = s <> getInfo columnDefinitionType
              columnDefinitionExtra = Nothing  -- TODO
              columnDefinitionNull = Nothing
              columnDefinitionDefault = Nothing
              columnDefinitionName = QColumnName s None name
  
          pure ColumnDefinition{..}
  
      partitionedByP = do
          _ <- Tok.partitionedP
          _ <- Tok.byP
          _ <- Tok.openP
          _ <- columnDefinitionP `sepBy1` Tok.commaP
          Tok.closeP
  
      clusteredByP = do
          _ <- Tok.clusteredP
          _ <- Tok.byP
          _ <- Tok.openP
          _ <- Tok.columnNameP `sepBy1` Tok.commaP
          _ <- Tok.closeP
          optional $ do
              _ <- Tok.sortedP
              _ <- Tok.byP
              _ <- Tok.openP
              _ <- (Tok.columnNameP >> optional directionP) `sepBy1` Tok.commaP
              Tok.closeP
          _ <- Tok.intoP
          _ <- Tok.numberP
          Tok.bucketsP
  
      serdeP = do
          _ <- Tok.serdeP
          e <- snd <$> Tok.stringP
          option e $ do
              _ <- Tok.withP
              _ <- Tok.serdePropertiesP
              _ <- Tok.openP
              _ <- propertyP `sepBy1` Tok.commaP
              Tok.closeP
  
      rowFormatP = do
          _ <- Tok.rowP
          _ <- Tok.formatP
          delimitedP <|> serdeP
  
      createTableColumnsP = do
          s <- Tok.openP
          c:cs <- (ColumnOrConstraintColumn <$> columnDefinitionP) `sepBy1` Tok.commaP
          e <- Tok.closeP
          pure $ TableColumns (s <> e) (c:|cs)
  
      createTableAsP = do
          s <- Tok.asP
          with <- option id withP
          query <- queryP (with, noInversion)  -- WITH *is* permitted in a CTAS. Inverted FROM is *not*.
          pure $ TableAs (s <> getInfo query) Nothing query
  
      createTableNoColumnInfoP r =
          -- r represents 'the point at which we decided there was no column info'
          pure $ TableNoColumnInfo r
  
  
  tblPropertiesP :: Parser (HiveMetadataProperties Range)
  tblPropertiesP = do
          s <- Tok.tblPropertiesP
          _ <- Tok.openP
          l <- propertyP `sepBy1` Tok.commaP
          e <- Tok.closeP
          let hiveMetadataPropertiesInfo = s <> e
              hiveMetadataPropertiesProperties = l
          pure $ HiveMetadataProperties{..}
  
  
  delimitedP :: Parser Range
  delimitedP = do
      s <- Tok.delimitedP
      e <- consumeOrderedOptions s $
          [ do
                _ <- Tok.fieldsP
                e' <- terminatedByCharP
                option e' $ do
                    _ <- Tok.escapedP
                    _ <- Tok.byP
                    snd <$> Tok.stringP
          , do
                _ <- Tok.collectionP
                _ <- Tok.itemsP
                terminatedByCharP
          , do
                _ <- Tok.mapP
                _ <- Tok.keysP
                terminatedByCharP
          , do
                _ <- Tok.linesP
                terminatedByCharP
          , do
                _ <- Tok.nullP
                _ <- Tok.definedP
                _ <- Tok.asP
                snd <$> Tok.stringP
            ]
      return $ s <> e
    where
      terminatedByCharP = do
          _ <- Tok.terminatedP
          _ <- Tok.byP
          snd <$> Tok.stringP
  
  
  
  ifExistsP :: Parser Range
  ifExistsP = do
      s <- Tok.ifP
      e <- Tok.existsP
      pure $ s <> e
  
  
  dropTableP :: Parser (DropTable RawNames Range)
  dropTableP = do
      s <- Tok.dropP
      _ <- Tok.tableP
      dropTableIfExists <- optionMaybe ifExistsP
      dropTableName <- tableNameP
      purge <- optionMaybe Tok.purgeP
  
      let dropTableInfo = s <> (fromMaybe (getInfo dropTableName) purge)
          dropTableNames = dropTableName :| []
      pure DropTable{..}
  
  
  alterTableRenameTablePrefixP :: Parser (Range, TableName RawNames Range)
  alterTableRenameTablePrefixP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      from <- tableNameP
      _ <- Tok.renameP
      pure $ (s, from)
  
  
  alterTableRenameTableP :: Parser (AlterTable RawNames Range)
  alterTableRenameTableP = do
      (s, from) <- alterTableRenameTablePrefixP
      _ <- Tok.toP
      to <- tableNameP
  
      pure $ AlterTableRenameTable (s <> getInfo to) from to
  
  
  alterTableRenameColumnPrefixP :: Parser (Range, TableName RawNames Range)
  alterTableRenameColumnPrefixP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      table <- tableNameP
      optional $ Tok.partitionP >> staticPartitionSpecP
      _ <- Tok.changeP
      pure (s, table)
  
  unqualifiedColumnNameP :: Parser (UQColumnName Range)
  unqualifiedColumnNameP = do
      (col, r) <- Tok.columnNameP
      pure $ QColumnName r None col
  
  alterTableRenameColumnP :: Parser (AlterTable RawNames Range)
  alterTableRenameColumnP = do
      (s, table) <- alterTableRenameColumnPrefixP
      optional Tok.columnP
      from <- unqualifiedColumnNameP
      to <- unqualifiedColumnNameP
      e <- getInfo <$> dataTypeP
      e' <- consumeOrderedOptions e $
                [ commentP
                , choice [ Tok.firstP
                         , Tok.afterP >> snd <$> Tok.columnNameP
                         ]
                , Tok.cascadeP <|> Tok.restrictP
                ]
      pure $ AlterTableRenameColumn (s <> e') table from to
  
  alterTableAddColumnsPrefixP :: Parser (Range, TableName RawNames Range)
  alterTableAddColumnsPrefixP = do
      s <- Tok.alterP
      _ <- Tok.tableP
      table <- tableNameP
      -- Note that Hive has ALTER TABLE ADD COLUMN for single partitions
      -- (!). Current behavior is to treat it the same as a top level ALTER TABLE
      -- ADD COLUMN. That seems fine initially, but hard to say if it makes sense
      -- in the long term.
      optional $ Tok.partitionP >> partitionSpecP
      _ <- Tok.addP
      e <- Tok.columnsP
      pure (s <> e, table)
  
  alterTableAddColumnsP :: Parser (AlterTable RawNames Range)
  alterTableAddColumnsP = do
      (s, table) <- alterTableAddColumnsPrefixP
      _ <- Tok.openP
      c:cs <- (colP `sepBy1` Tok.commaP)
      e <- Tok.closeP
      e' <- option e (Tok.cascadeP <|> Tok.restrictP)
      pure $ AlterTableAddColumns (s <> e') table (c:|cs)
    where
      colP :: Parser (UQColumnName Range)
      colP = do
          col <- unqualifiedColumnNameP
          _ <- dataTypeP
          _ <- optional commentP
          return col
  
  grantP :: Parser (Grant Range)
  grantP = do
      s <- Tok.grantP
      e <- P.many1 Tok.notSemicolonP
      return $ Grant (s <> (last e))
  
  revokeP :: Parser (Revoke Range)
  revokeP = do
      s <- Tok.revokeP
      e <- P.many1 Tok.notSemicolonP
      return $ Revoke (s <> (last e))
  
  integerP :: Parser (Int, Range)
  integerP = do
      (n, e) <- Tok.numberP
      case reads $ TL.unpack n of
          [(n', "")] -> pure (n', e)
          _ -> fail $ unwords ["unable to parse", show n, "as integer"]
  
  
  countingSepBy1 :: (Integer -> Parser b) -> Parser c -> Parser [b]
  countingSepBy1 f sep = do
      x <- f 0
      xs <- rest 1
      pure (x:xs)
    where
      rest n = choice
          [ do
              _ <- sep
              x <- f n
              xs <- rest (n + 1)
              pure (x:xs)
          , pure []
          ]
  
  
  introduceAliases :: Set Text -> ParserScope -> ParserScope
  introduceAliases aliases = \ scope ->
      let unioned = case selectTableAliases scope of
              Nothing -> aliases
              Just existing -> S.union existing aliases
      in scope { selectTableAliases = Just unioned }
  
  tablishToTableAlias :: Tablish RawNames Range -> Set Text
  tablishToTableAlias (TablishTable _ aliases tableName) = case aliases of
      TablishAliasesNone -> tableNameToTableAlias tableName
      TablishAliasesT (TableAlias _ name _) -> S.singleton name
      TablishAliasesTC _ _ -> error "shouldn't happen in hive"
  tablishToTableAlias (TablishSubQuery _ aliases _) = case aliases of
      TablishAliasesNone -> error "shouldn't happen in hive"
      TablishAliasesT (TableAlias _ name _) -> S.singleton name
      TablishAliasesTC _ _ -> error "shouldn't happen in hive"
  tablishToTableAlias (TablishLateralView _ LateralView{..} _) = case lateralViewAliases of
      TablishAliasesNone -> error "shouldn't happen in hive"
      TablishAliasesT (TableAlias _ name _) -> S.singleton name
      TablishAliasesTC (TableAlias _ name _) _ -> S.singleton name
  tablishToTableAlias (TablishJoin _ (JoinSemi _) _ lTablish _) =
      tablishToTableAlias lTablish
  tablishToTableAlias (TablishJoin _ _ _ lTablish rTablish) =
      tablishToTableAlias lTablish `S.union` tablishToTableAlias rTablish
  
  tableNameToTableAlias :: OQTableName Range -> Set Text
  tableNameToTableAlias (QTableName _ _ name) = S.singleton name
  
  
  fromP :: Parser (SelectFrom RawNames Range)
  fromP = do
      r <- Tok.fromP
      tablishes <- tablishP `sepBy1` Tok.commaP
  
      let r' = foldl (<>) r $ fmap getInfo tablishes
      return $ SelectFrom r' tablishes
  
  type InvertedFrom = Maybe (SelectFrom RawNames Range)
  
  noInversion :: InvertedFrom
  noInversion = Nothing
  
  invertedFromP :: Parser InvertedFrom
  invertedFromP = optionMaybe fromP
  
  
  selectP :: InvertedFrom -> Parser (Select RawNames Range)
  selectP invertedFrom = do
      r <- Tok.selectP
  
      selectDistinct <- option notDistinct distinctP
  
      aliases <- try $ selectScopeLookAhead invertedFrom
  
      selectCols <- do
          selections <- local (introduceAliases aliases) $ selectionP `countingSepBy1` Tok.commaP
          let r' = foldl1 (<>) $ map getInfo selections
          return $ SelectColumns r' selections
  
      selectFrom <- maybe (optionMaybe fromP) (return . Just) invertedFrom
      selectWhere <- optionMaybe $ local (introduceAliases aliases) whereP
      let selectTimeseries = Nothing
      selectGroup <- optionMaybe selectGroupP
      selectHaving <- optionMaybe havingP
      selectNamedWindow <- optionMaybe namedWindowP
  
      let (Just selectInfo) = sconcat $ Just r :|
              [ Just $ getInfo selectCols
              , getInfo <$> selectFrom
              , getInfo <$> selectWhere
              , getInfo <$> selectGroup
              , getInfo <$> selectHaving
              , getInfo <$> selectNamedWindow
              ]
      return Select{..}
  
    where
      selectScopeLookAhead :: InvertedFrom -> Parser (Set Text)
      selectScopeLookAhead invertedFrom' = P.lookAhead $ do
          _ <- selectionP (-1) `sepBy1` Tok.commaP
          from <- maybe (optionMaybe fromP) (return . Just) invertedFrom'
  
          let tablishes = case from of
                  Just (SelectFrom _ ts) -> ts
                  Nothing -> []
              aliases = L.foldl' S.union S.empty $ map tablishToTableAlias tablishes
          return aliases
  
      whereP = do
          r <- Tok.whereP
          condition <- exprP
          return $ SelectWhere (r <> getInfo condition) condition
  
      havingP = do
          r <- Tok.havingP
          conditions <- exprP `sepBy1` Tok.commaP
  
          let r' = foldl (<>) r $ fmap getInfo conditions
          return $ SelectHaving r' conditions
  
      namedWindowP =
        do
          r <- Tok.windowP
          windows <- (flip sepBy1) Tok.commaP $ do
            name <- windowNameP
            _ <- Tok.asP
            s <- Tok.openP
            window <- choice
                [ do
                    partition <- optionMaybe partitionP
                    order <- option [] orderInWindowClauseP
                    frame <- optionMaybe frameP
                    e <- Tok.closeP
                    let info = s <> e
                    return $ Left $ WindowExpr info partition order frame
                , do
                    inherit <- windowNameP
                    partition <- optionMaybe partitionP
                    order <- option [] orderInWindowClauseP
                    frame <- optionMaybe frameP
                    e <- Tok.closeP
                    let info = s <> e
                    return $ Right $ PartialWindowExpr info inherit partition order frame
                ]
  
            let infof = (getInfo name <>)
            return $ case window of
              Left w -> NamedWindowExpr (infof $ getInfo w) name w
              Right pw -> NamedPartialWindowExpr (infof $ getInfo pw) name pw
          let info = L.foldl' (<>) r $ fmap getInfo windows
          return $ SelectNamedWindow info windows
  
  
  handlePositionalReferences :: Expr RawNames Range -> PositionOrExpr RawNames Range
  handlePositionalReferences e = case e of
      ConstantExpr _ (NumericConstant _ n) | TL.all isDigit n -> PositionOrExprPosition (getInfo e) (read $ TL.unpack n) Unused
      _ -> PositionOrExprExpr e
  
  selectGroupP :: Parser (SelectGroup RawNames Range)
  selectGroupP = do
      r <- Tok.groupP
      _ <- Tok.byP
  
      rawExprs <- exprP `sepBy1` Tok.commaP
      let exprs = map (toGroupingElement . handlePositionalReferences) rawExprs
  
      sets <- option [] $ choice
          [ groupingSetsP
          , do
              _ <- try $ P.lookAhead $ Tok.withP >> Tok.cubeP
              cubeP rawExprs
          , do
              _ <- try $ P.lookAhead $ Tok.withP >> Tok.rollupP
              rollupP rawExprs
          ]
  
      let selectGroupGroupingElements = exprs ++ sets
          selectGroupInfo = foldl (<>) r $ fmap getInfo selectGroupGroupingElements
  
      return SelectGroup{..}
    where
      toGroupingElement :: PositionOrExpr RawNames Range -> GroupingElement RawNames Range
      toGroupingElement posOrExpr = GroupingElementExpr (getInfo posOrExpr) posOrExpr
  
      groupingSetP :: Parser (GroupingElement RawNames Range)
      groupingSetP = choice $
          [ do
                s <- Tok.openP
                sets <- exprP `sepBy` Tok.commaP
                e <- Tok.closeP
                return $ GroupingElementSet (s <> e) sets
          , do
                -- if no parens, it will be the singleton list.
                expr <- exprP
                return $ GroupingElementSet (getInfo expr) [expr]
          ]
  
      groupingSetsP :: Parser [GroupingElement RawNames Range]
      groupingSetsP = do
          _ <- Tok.groupingP
          _ <- Tok.setsP
          _ <- Tok.openP
          sets <- groupingSetP `sepBy1` Tok.commaP
          _ <- Tok.closeP
          return sets
  
      toGroupingSet :: Range -> [Expr RawNames Range] -> GroupingElement RawNames Range
      toGroupingSet r [] = GroupingElementSet r []
      toGroupingSet _ exprs =
          let s = getInfo $ head exprs
              e = getInfo $ last exprs
           in GroupingElementSet (s <> e) exprs
  
      cubeP :: [Expr RawNames Range] -> Parser [GroupingElement RawNames Range]
      cubeP exprs = do
          _ <- Tok.withP
          _ <- Tok.cubeP
          let dimensions = L.subsequences exprs
              defaultRange = (getInfo $ head exprs) <> (getInfo $ last exprs)
          return $ map (toGroupingSet defaultRange) dimensions
  
      rollupP :: [Expr RawNames Range] -> Parser [GroupingElement RawNames Range]
      rollupP exprs = do
          _ <- Tok.withP
          _ <- Tok.rollupP
          let dimensions = L.reverse $ L.inits exprs
              defaultRange = (getInfo $ head exprs) <> (getInfo $ last exprs)
          return $ map (toGroupingSet defaultRange) dimensions
  
  
  -- | 'selectClusterP' parses for either clusterby or distributeby/sortby
  -- T478023 - implement clusterby in select datatype
  selectClusterP :: Parser ()
  selectClusterP = choice
      [ clusterP
      , distributeSortP
      ]
  
    where
      clusterP :: Parser ()
      clusterP =
        do
          _ <- Tok.clusterP
          _ <- Tok.byP
          _ <- sepBy1 exprP Tok.commaP
          return ()
  
      distributeSortP :: Parser ()
      distributeSortP =
        do
          optional distributeP
          optional sortP
  
      distributeP :: Parser ()
      distributeP =
        do
          _ <- Tok.distributeP
          _ <- Tok.byP
          _ <- sepBy1 exprP Tok.commaP
          return ()
  
      sortP :: Parser ()
      sortP =
        do
          _ <- Tok.sortP
          _ <- Tok.byP
          _ <- flip sepBy1 Tok.commaP $ do
            expr <- exprP
            direction <- option (OrderAsc Nothing) $ choice
                [ OrderAsc . Just <$> Tok.ascP
                , OrderDesc . Just <$> Tok.descP
                ]
            return (expr, direction)
          return ()
  
  
  qualifiedTableNameP :: Parser (Text, Text, Range, Range)
  qualifiedTableNameP = do
      (s, r) <- Tok.schemaNameP
      _ <- Tok.dotP
      (t, r') <- Tok.tableNameP
  
      return (s, t, r, r')
  
  
  -- | The columnName parser has been overhauled with checks for table names. If
  -- a scope is present (i.e. in a select statement), the table name must be
  -- a member of the tableAlias list for the parser to succeed. Otherwise,
  -- the table parser fails and execution tries the next parser choice.
  --
  -- Should the scope not be set, e.g. when selectP is performing lookahead
  -- to build scope, this check is skipped.
  checkTableNameInScopeP :: Text -> Parser ()
  checkTableNameInScopeP name = do
      maybeScope <- asks selectTableAliases
      case maybeScope of
          Just scope ->
              case L.find (==name) scope of
                  Just _  -> return ()
                  Nothing -> fail $ "Table " ++ (show name) ++
                      " doesn't exist in table scope " ++ show maybeScope
          Nothing -> return ()
  
  
  selectStarP :: Parser (Selection RawNames Range)
  selectStarP = choice
      [ do
          r <- Tok.starP
          return $ SelectStar r Nothing Unused
  
      , try $ do
          (t, r) <- Tok.tableNameP
          _ <- Tok.dotP
          r' <- Tok.starP
  
          return $ SelectStar (r <> r') (Just $ QTableName r Nothing t) Unused
  
      , try $ do
          (s, t, r, r') <- qualifiedTableNameP
          _ <- Tok.dotP
          r'' <- Tok.starP
  
          return $ SelectStar (r <> r'')
              (Just $ QTableName r' (Just $ mkNormalSchema s r) t) Unused
      ]
  
  
  tableNameP :: Parser (OQTableName Range)
  tableNameP = choice
      [ try $ do
          (s, t, r, r') <- qualifiedTableNameP
          return $ QTableName r' (Just $ mkNormalSchema s r) t
  
      , do
          (t, r) <- Tok.tableNameP
          return $ QTableName r Nothing t
      ]
  
  
  arrayAccessP :: Parser (Expr RawNames Range -> Expr RawNames Range)
  arrayAccessP = do
      _ <- Tok.openBracketP
      index <- exprP
      e <- Tok.closeBracketP
      return $ \ expr ->
          let exprR = getInfo expr <> e
          in ArrayAccessExpr exprR expr index
  
  
  structFieldNameP :: Parser (StructFieldName Range)
  structFieldNameP = do
      (t, r) <- Tok.structFieldNameP
      return $ StructFieldName r t
  
  
  structAccessP :: Parser (Expr RawNames Range -> Expr RawNames Range)
  structAccessP = do
      _ <- Tok.dotP
      field <- structFieldNameP
      return $ \ struct ->
          let r = getInfo struct <> getInfo field
           in FieldAccessExpr r struct field
  
  
  columnNameP :: Parser (OQColumnName Range)
  columnNameP = choice
      -- Note that in hive, column names cannot lead with schema qualifiers
      [ try $ do
          (t, r) <- Tok.tableNameP
          _ <- Tok.dotP
          (c, r') <- Tok.columnNameP
  
          _ <- checkTableNameInScopeP t
  
          return $ QColumnName r' (Just $ QTableName r Nothing t) c
  
      , do
          (c, r) <- Tok.columnNameP
  
          return $ QColumnName r Nothing c
      ]
  
  
  selectionP :: Integer -> Parser (Selection RawNames Range)
  selectionP idx = try selectStarP <|> do
      expr <- exprP
      aliases <- aliasesP expr idx
      let info = foldr (<>) (getInfo expr) (map getInfo aliases)
  
      return $ SelectExpr info aliases expr
    where
      aliasesP :: Expr RawNames Range -> Integer -> Parser [ColumnAlias Range]
      aliasesP expr idx' = choice
          [ try $ do
              optional Tok.asP
              (name, r) <- Tok.columnNameP -- SELECT 1 as foo FROM bar;
              pure <$> makeColumnAlias r name
  
          , try $ do
              _ <- Tok.asP
              P.between Tok.openP Tok.closeP $ flip sepBy1 Tok.commaP $ do
                  (name, r) <- Tok.columnNameP -- SELECT <expr> as (foo1, foo2) FROM bar;
                  makeColumnAlias r name
  
          , do
              r <- Tok.asP
              pure <$> makeColumnAlias r "as" -- SELECT 1 as FROM bar;  i.e. `as` is the alias!
  
          , pure <$> makeExprAlias expr idx'
          ]
  
  
  makeColumnAlias :: Range -> Text -> Parser (ColumnAlias Range)
  makeColumnAlias r alias = ColumnAlias r alias . ColumnAliasId <$> getNextCounter
  
  makeTableAlias :: Range -> Text -> Parser (TableAlias Range)
  makeTableAlias r alias = TableAlias r alias . TableAliasId <$> getNextCounter
  
  makeDummyAlias :: Range -> Integer -> Parser (ColumnAlias Range)
  makeDummyAlias r idx = makeColumnAlias r $ TL.pack $ "_c" ++ show idx
  
  makeExprAlias :: Expr RawNames Range -> Integer -> Parser (ColumnAlias Range)
  makeExprAlias (ColumnExpr info (QColumnName _ _ name)) _ = makeColumnAlias info name
  makeExprAlias expr idx = makeDummyAlias (getInfo expr) idx
  
  
  exprP :: Parser (Expr RawNames Range)
  exprP = orExprP
  
  parenExprP :: Parser (Expr RawNames Range)
  parenExprP = P.between Tok.openP Tok.closeP exprP
  
  caseExprP :: Parser (Expr RawNames Range)
  caseExprP = do
      r <- Tok.caseP
      whens <- choice
          [ P.many1 $ do
              _ <- Tok.whenP
              condition <- exprP
              _ <- Tok.thenP
              result <- exprP
              return (condition, result)
  
          , do
              expr <- exprP
              P.many1 $ do
                  whenr <- Tok.whenP
                  condition <- BinOpExpr whenr "=" expr <$> exprP
                  _ <- Tok.thenP
                  result <- exprP
                  return (condition, result)
          ]
  
      melse <- optionMaybe $ do
          _ <- Tok.elseP
          exprP
  
      r' <- Tok.endP
  
      return $ CaseExpr (r <> r') whens melse
  
  
  fieldTypeP :: Parser (Expr RawNames Range)
  fieldTypeP = do
      (ftype, r) <- Tok.fieldTypeP
      return $ ConstantExpr r $ StringConstant r $ TL.encodeUtf8 ftype
  
  functionExprP :: Parser (Expr RawNames Range)
  functionExprP = choice
      [ castFuncP
      , dateDiffFuncP
      , extractFuncP
      , try regularFuncP
      , bareFuncP
      ]
    where
      castFuncP = do
          r <- Tok.castP
          _ <- Tok.openP
          e <- exprP
          _ <- Tok.asP
          t <- dataTypeP
          r' <- Tok.closeP
  
          return $ TypeCastExpr (r <> r') CastFailureError e t
  
      dateDiffFuncP = do
          r <- Tok.dateDiffP
          _ <- Tok.openP
  
          date1 <- exprP
          _ <- Tok.commaP
          date2 <- exprP
          r' <- Tok.closeP
  
          return $ FunctionExpr (r <> r') (QFunctionName r Nothing "datediff") notDistinct [date1, date2] [] Nothing Nothing
  
      extractFuncP = do
          r <- Tok.extractP
          _ <- Tok.openP
          ftype <- fieldTypeP
          _ <- Tok.fromP
          expr <- exprP
          r' <- Tok.closeP
  
          return $ FunctionExpr (r <> r') (QFunctionName r Nothing "extract") notDistinct [ftype, expr] [] Nothing Nothing
  
      regularFuncP = do
          name <- choice
              [ try $ do
                  (s, r) <- Tok.schemaNameP
                  _ <- Tok.dotP
                  (f, r') <- Tok.functionNameP
                  return $ QFunctionName (r <> r') (Just $ mkNormalSchema s r) f
  
              , do
                  (f, r) <- Tok.functionNameP
                  return $ QFunctionName r Nothing f
              ]
  
          (distinct, arguments, parameters, r') <- do
              _ <- Tok.openP
              (distinct, arguments) <- choice
                  [ case name of
                      QFunctionName _ Nothing "count" -> do
                          r' <- Tok.starP
                          return ( notDistinct
                                 , [ConstantExpr r' $ NumericConstant r' "1"]
                                 )
  
                      _ -> fail "not count, can't star"
  
                  , do
                      isDistinct <- distinctP
                      (isDistinct,) . (:[]) <$> exprP
  
                  , (notDistinct,) <$> exprP `sepBy` Tok.commaP
                  ]
  
              optional $ Tok.ignoreP >> Tok.nullsP
  
              r' <- Tok.closeP
              return (distinct, arguments, [], r')
  
          over <- optionMaybe $ try $ overP
  
          let r'' = maybe r' getInfo over <> getInfo name
  
          return $ FunctionExpr r'' name distinct arguments parameters Nothing over
  
      bareFuncP = do
          (v, r) <- choice
              [ Tok.currentDatabaseP
              , Tok.currentSchemaP
              , Tok.currentUserP
              , Tok.sessionUserP
              , Tok.currentDateP
              , Tok.currentTimeP
              , Tok.currentTimestampP
              , Tok.localTimeP
              , Tok.localTimestampP
              , Tok.sysDateP
              ]
  
          pure $ FunctionExpr r (QFunctionName r Nothing v) notDistinct [] [] Nothing Nothing
  
  orderTopLevelP :: Parser (Range, [Order RawNames Range])
  orderTopLevelP = orderExprP False True
  
  orderInWindowClauseP :: Parser [Order RawNames Range]
  orderInWindowClauseP = snd <$> orderExprP True False
  
  orderExprP :: Bool -> Bool -> Parser (Range, [Order RawNames Range])
  orderExprP nullsClausePermitted positionalReferencesPermitted = do
      r <- Tok.orderP
      _ <- Tok.byP
      orders <- helperP `sepBy1` Tok.commaP
      let r' = getInfo $ last orders
      return (r <> r', orders)
    where
      helperP :: Parser (Order RawNames Range)
      helperP = do
          expr <- exprP
          let posOrExpr = if positionalReferencesPermitted
                          then handlePositionalReferences expr
                          else PositionOrExprExpr expr
          dir <- directionP
          nulls <- case (nullsClausePermitted, dir) of
              (False, _) -> return $ NullsAuto Nothing
              (True, OrderAsc _) -> option (NullsLast Nothing) nullsP
              (True, OrderDesc _) -> option (NullsFirst Nothing) nullsP
          let info = (getInfo expr) ?<> (getInfo dir) <> (getInfo nulls)
          return $ Order info posOrExpr dir nulls
  
  directionP :: Parser (OrderDirection (Maybe Range))
  directionP = option (OrderAsc Nothing) $ choice
      [ OrderAsc . Just <$> Tok.ascP
      , OrderDesc . Just <$> Tok.descP
      ]
  
  nullsP :: Parser (NullPosition (Maybe Range))
  nullsP = do
      r <- Tok.nullsP
      choice
          [ Tok.firstP >>= \ r' -> return $ NullsFirst $ Just $ r <> r'
          , Tok.lastP >>= \ r' -> return $ NullsLast $ Just $ r <> r'
          , Tok.autoP >>= \ r' -> return $ NullsAuto $ Just $ r <> r'
          ]
  
  frameP :: Parser (Frame Range)
  frameP = do
      ftype <- choice
          [ RowFrame <$> Tok.rowsP
          , RangeFrame <$> Tok.rangeP
          ]
  
      choice
          [ do
              _ <- Tok.betweenP
              start <- frameBoundP
              _ <- Tok.andP
              end <- frameBoundP
  
              let r = getInfo ftype <> getInfo end
              return $ Frame r ftype start (Just end)
  
          , do
              start <- frameBoundP
  
              let r = getInfo ftype <> getInfo start
              return $ Frame r ftype start Nothing
          ]
  
  frameBoundP :: Parser (FrameBound Range)
  frameBoundP = choice
      [ fmap Unbounded $ (<>)
          <$> Tok.unboundedP
          <*> choice [ Tok.precedingP, Tok.followingP ]
  
      , fmap CurrentRow $ (<>) <$> Tok.currentP <*> Tok.rowP
      , constantP >>= \ expr -> choice
          [ Tok.precedingP >>= \ r ->
              return $ Preceding (getInfo expr <> r) expr
  
          , Tok.followingP >>= \ r ->
              return $ Following (getInfo expr <> r) expr
          ]
      ]
  
  
  overP :: Parser (OverSubExpr RawNames Range)
  overP = do
      start <- Tok.overP
      subExpr <- choice
          [ Left <$> windowP
          , Right <$> windowNameP
          ]
      return $ case subExpr of
        Left w -> mergeWindowInfo start w
        Right wn -> OverWindowName (start <> getInfo wn) wn
    where
      windowP :: Parser (OverSubExpr RawNames Range)
      windowP = do
        start' <- Tok.openP
        expr <- choice
            [ Left <$> windowExprP start'
            , Right <$> partialWindowExprP start'
            ]
        return $ case expr of
          Left w -> OverWindowExpr (start' <> getInfo w) w
          Right pw -> OverPartialWindowExpr (start' <> getInfo pw) pw
  
      mergeWindowInfo :: Range -> OverSubExpr RawNames Range -> OverSubExpr RawNames Range
      mergeWindowInfo r = \case
          OverWindowExpr r' WindowExpr{..} ->
              OverWindowExpr (r <> r') $ WindowExpr { windowExprInfo = windowExprInfo <> r , ..}
          OverWindowName r' n -> OverWindowName (r <> r') n
          OverPartialWindowExpr r' PartialWindowExpr{..} ->
              OverPartialWindowExpr (r <> r') $ PartialWindowExpr { partWindowExprInfo = partWindowExprInfo <> r , ..}
  
  windowExprP :: Range -> Parser (WindowExpr RawNames Range)
  windowExprP start =
    do
      partition <- optionMaybe partitionP
      order <- option [] orderInWindowClauseP
      frame <- optionMaybe frameP
      end <- Tok.closeP
      let info = start <> end
      return (WindowExpr info partition order frame)
  
  partialWindowExprP :: Range -> Parser (PartialWindowExpr RawNames Range)
  partialWindowExprP start =
    do
      inherit <- windowNameP
      partition <- optionMaybe partitionP
      order <- option [] orderInWindowClauseP
      frame <- optionMaybe frameP
      end <- Tok.closeP
      let info = start <> end
      return (PartialWindowExpr info inherit partition order frame)
  
  windowNameP :: Parser (WindowName Range)
  windowNameP =
    do
      (name, r) <- Tok.windowNameP
      return $ WindowName r name
  
  partitionP :: Parser (Partition RawNames Range)
  partitionP = do
      r <- Tok.partitionP
      choice
          [ do
                _ <- Tok.byP
                exprs <- optionalParensP $ exprP `sepBy1` Tok.commaP
                return $ PartitionBy (sconcat $ r :| map getInfo exprs) exprs
  
          , Tok.bestP >>= \ r' -> return $ PartitionBest (r <> r')
          , Tok.nodesP >>= \ r' -> return $ PartitionNodes (r <> r')
          ]
  
  
  dataTypeP :: Parser (DataType Range)
  dataTypeP = choice
      [ arrayTypeP
      , mapTypeP
      , structTypeP
      , unionTypeP
      , primitiveTypeP
      ]
    where
      primitiveTypeP = do
          (name, r) <- Tok.typeNameP
          args <- option [] $ P.between Tok.openP Tok.closeP $ constantP `sepBy1` Tok.commaP
          return $ PrimitiveDataType r name $ map DataTypeParamConstant args
  
      arrayTypeP = do
          s <- Tok.arrayP
          _ <- Tok.openAngleP
          itemType <- dataTypeP
          e <- Tok.closeAngleP
          return $ ArrayDataType (s <> e) itemType
  
      mapTypeP = do
          s <- Tok.mapP
          _ <- Tok.openAngleP
          keyType <- primitiveTypeP
          _ <- Tok.commaP
          valueType <- dataTypeP
          e <- Tok.closeAngleP
          return $ MapDataType (s <> e) keyType valueType
  
      structTypeP = do
          s <- Tok.structP
          _ <- Tok.openAngleP
          let fieldP = do
                (name, _) <- Tok.structFieldNameP
                _ <- Tok.colonP
                type_ <- dataTypeP
                optional commentP
                return (name, type_)
          fields <- fieldP `sepBy1` Tok.commaP
          e <- Tok.closeAngleP
          return $ StructDataType (s <> e) fields
  
      unionTypeP = do
          s <- Tok.uniontypeP
          _ <- Tok.openAngleP
          types <- dataTypeP `sepBy1` Tok.commaP
          e <- Tok.closeAngleP
          return $ UnionDataType (s <> e) types
  
  
  existsExprP :: Parser (Expr RawNames Range)
  existsExprP = do
      r <- Tok.existsP
      _ <- Tok.openP
      query <- querySelectP (emptyPrefix, noInversion)
      r' <- Tok.closeP
  
      return $ ExistsExpr (r <> r') query
  
  
  columnExprP :: Parser (Expr RawNames Range)
  columnExprP = do
      col <- columnNameP
      return $ ColumnExpr (getInfo col) col
  
  
  variableSubstitutionP :: Parser (Expr RawNames Range)
  variableSubstitutionP = do
      info <- Tok.variableSubstitutionP
      return $ VariableSubstitutionExpr info
  
  
  exprWithArrayOrStructAccessP :: Parser (Expr RawNames Range)
  exprWithArrayOrStructAccessP = foldl (flip ($)) <$> baseP <*> many (structAccessP <|> arrayAccessP)
    where
      baseP :: Parser (Expr RawNames Range)
      baseP = choice
          [ try parenExprP
          , try existsExprP
          , try functionExprP
          , caseExprP
          , try $ do
              constant <- constantP
              return $ ConstantExpr (getInfo constant) constant
  
          , columnExprP
          , variableSubstitutionP
          ]
  
  
  unOpP :: Text -> Parser (Expr RawNames Range -> Expr RawNames Range)
  unOpP op = do
      r <- Tok.symbolP op
      return $ \ expr -> UnOpExpr (r <> getInfo expr) (Operator op) expr
  
  
  unaryPrefixExprP :: Parser (Expr RawNames Range)
  unaryPrefixExprP = do
      prefix <- option id $ choice $ map unOpP [ "+", "-", "~" ]
      expr <- exprWithArrayOrStructAccessP
      return $ prefix expr
  
  
  notOperatorP :: Parser (Expr RawNames Range -> Expr RawNames Range)
  notOperatorP = (\ r -> UnOpExpr r "NOT") <$> Tok.notOperatorP
  
  unarySuffixExprP :: Parser (Expr RawNames Range)
  unarySuffixExprP = do
      expr <- unaryPrefixExprP
      is <- option id $ do
              _ <- Tok.isP
              not_ <- option id notOperatorP
              (not_ .) <$> (Tok.nullP >>= \ r -> return (UnOpExpr r "ISNULL"))
  
      return $ is expr
  
  
  binOpP :: Text -> Parser (Expr RawNames Range -> Expr RawNames Range -> Expr RawNames Range)
  binOpP op = do
      r <- Tok.symbolP op
  
      let r' lhs rhs = sconcat $ r :| map getInfo [lhs, rhs]
      return $ \ lhs rhs -> BinOpExpr (r' lhs rhs) (Operator op) lhs rhs
  
  
  bitwiseXorExprP :: Parser (Expr RawNames Range)
  bitwiseXorExprP = unarySuffixExprP `chainl1` binOpP "^"
  
  
  productExprP :: Parser (Expr RawNames Range)
  productExprP = bitwiseXorExprP `chainl1` opP
    where
      opP = choice $ map binOpP [ "*", "/", "%" ]
  
  
  sumExprP :: Parser (Expr RawNames Range)
  sumExprP = productExprP `chainl1` opP
    where
      opP = choice $ map binOpP [ "+", "-" ]
  
  
  stringExprP :: Parser (Expr RawNames Range)
  stringExprP = sumExprP `chainl1` opP
    where
      opP = choice $ map binOpP [ "||" ]
  
  
  bitwiseAndExprP :: Parser (Expr RawNames Range)
  bitwiseAndExprP = stringExprP `chainl1` binOpP "&"
  
  
  bitwiseOrExprP :: Parser (Expr RawNames Range)
  bitwiseOrExprP = bitwiseAndExprP `chainl1` binOpP "|"
  
  
  inExprP :: Parser (Expr RawNames Range)
  inExprP = do
      expr <- bitwiseOrExprP
      not_ <- option id notOperatorP
      in_ <- foldl (.) id <$> many inP
  
      return $ not_ $ in_ expr
  
    where
      inP = do
          _ <- Tok.inP
          _ <- Tok.openP
          list <- choice
              [ Left <$> queryP (emptyPrefix, noInversion)
              , Right <$> exprP `sepBy1` Tok.commaP
              ]
  
          r <- Tok.closeP
  
          return $ case list of
              Left query ->
                  \ expr -> InSubqueryExpr (getInfo expr <> r) query expr
  
              Right constants ->
                  \ expr -> InListExpr (getInfo expr <> r) constants expr
  
  betweenExprP :: Parser (Expr RawNames Range)
  betweenExprP = do
      expr <- inExprP
      between <- foldl (.) id <$> many betweenP
  
      return $ between expr
    where
      betweenP = do
          _ <- Tok.betweenP
          start <- sumExprP
          _ <- Tok.andP
          end <- sumExprP
  
          let r expr = getInfo expr <> getInfo end
          return $ \ expr -> BetweenExpr (r expr) start end expr
  
  
  likeExprP :: Parser (Expr RawNames Range)
  likeExprP = do
      expr <- betweenExprP
      like <- option id comparisonP
      return $ like expr
    where
      comparisonP :: Parser (Expr RawNames Range -> Expr RawNames Range)
      comparisonP = do
        comparison <- textComparisonP
        pattern <- Pattern <$> betweenExprP
        return $ comparison Nothing pattern
  
      textComparisonP :: Parser (Maybe (Escape RawNames Range) -> Pattern RawNames Range -> Expr RawNames Range -> Expr RawNames Range)
      textComparisonP = do
          not_ <- option id notOperatorP
  
          like <- choice
              [ Tok.likeP   >>= \ r -> return $ LikeExpr r "LIKE"
              , Tok.rlikeP  >>= \ r -> return $ LikeExpr r "RLIKE"
              , Tok.regexpP >>= \ r -> return $ LikeExpr r "REGEXP"
              ]
  
          return $ \ escape pattern expr -> not_ $ like escape pattern expr
  
  
  mkBinOp :: (Text, a) -> Expr r a -> Expr r a -> Expr r a
  mkBinOp (op, r) = BinOpExpr r (Operator op)
  
  inequalityExprP :: Parser (Expr RawNames Range)
  inequalityExprP = likeExprP `chainl1` (mkBinOp <$> Tok.inequalityOpP)
  
  
  equalityExprP :: Parser (Expr RawNames Range)
  equalityExprP = inequalityExprP `chainl1` (mkBinOp <$> Tok.equalityOpP)
  
  
  notExprP :: Parser (Expr RawNames Range)
  notExprP = do
      not_ <- option id notOperatorP
      expr <- equalityExprP
      return $ not_ expr
  
  
  andExprP :: Parser (Expr RawNames Range)
  andExprP = notExprP `chainl1`
      (Tok.andP >>= \ r -> return $ BinOpExpr r "AND")
  
  
  orExprP :: Parser (Expr RawNames Range)
  orExprP = andExprP `chainl1` (Tok.orP >>= \ r -> return (BinOpExpr r "OR"))
  
  
  singleTableP :: Parser (Tablish RawNames Range)
  singleTableP = try subqueryP <|> try tableP
    where
      subqueryP = do
          r <- Tok.openP
          invertedFrom <- invertedFromP
          query <- queryP (emptyPrefix, invertedFrom)
          _ <- Tok.closeP
          maybe_alias <- aliasP
          case maybe_alias of
              Nothing -> fail $ "in hive, tablish subquery must have alias"
              Just alias -> return $ TablishSubQuery (r <> getInfo alias) (TablishAliasesT alias) query
  
      tableP = do
          name <- tableNameP
          _ <- optional tableSampleP
          maybe_alias <- aliasP
          let r = case maybe_alias of
                  Nothing -> getInfo name
                  Just alias -> getInfo alias <> getInfo name
              aliases = maybe TablishAliasesNone TablishAliasesT maybe_alias
  
          return $ TablishTable r aliases name
  
      -- This is slightly complicated because `full` is permitted as a table
      -- alias, but `full` is also a reserved word that may follow a table alias.
      -- Strategy is lookahead... if the next tokens are `full` and `outer` and
      -- `join`, we definitely don't have an alias so return Nothing. Otherwise,
      -- there may or may not be an alias.
      aliasP :: Parser (Maybe (TableAlias Range))
      aliasP = choice
          [ do
                _ <- try $ P.lookAhead $ Tok.fullP >> optional Tok.outerP >> Tok.joinP
                return Nothing
          , optionMaybe $ (optional Tok.asP) >> tableAliasP
          ]
  
      tableSampleP :: Parser Range
      tableSampleP = do
          s <- Tok.tableSampleP
          _ <- Tok.openP
          _ <- choice $
              [ do
                  s' <- Tok.bucketP
                  _ <- Tok.numberP
                  _ <- Tok.outP
                  _ <- Tok.ofP
                  _ <- Tok.numberP
                  option s' $ do
                      _ <- Tok.onP
                      choice $ [ try $ Tok.randP >> Tok.openP >> Tok.closeP
                               , snd <$> Tok.columnNameP
                               ]
              , Tok.numberP >> (Tok.percentP <|> Tok.rowsP)
              , snd <$> Tok.byteAmountP
              ]
          e <- Tok.closeP
          return $ s <> e
  
  singleTableWithViewsP :: Parser (Tablish RawNames Range)
  singleTableWithViewsP = do
      table <- singleTableP
      views <- fmap (appEndo . fold . reverse) $ many $ Endo <$> lateralViewP
      return $ views table
  
  
  -- see https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView
  lateralViewP :: Parser (Tablish RawNames Range -> Tablish RawNames Range)
  lateralViewP = do
      s <- Tok.lateralP
      _ <- Tok.viewP
      lateralViewOuter <- optionMaybe Tok.outerP
      lateralViewExprs <- (:[]) <$> functionExprP
      let lateralViewWithOrdinality = False -- it's not a thing in Hive
  
      tAlias <- tableAliasP
      cAliases <- optionMaybe $ do
          _ <- Tok.asP
          columnAliasP `sepBy1` Tok.commaP
      let lateralViewAliases = case cAliases of
              Nothing -> TablishAliasesT tAlias
              Just cAliases' -> TablishAliasesTC tAlias cAliases'
  
          e = getInfo tAlias
          es = maybe [] (map getInfo) cAliases
          lateralViewInfo = s <> sconcat (e:|es)
  
      pure $ \ lhs -> TablishLateralView (getInfo lhs <> lateralViewInfo) LateralView{..} (Just lhs)
  
  
  optionalParensP :: Parser a -> Parser a
  optionalParensP p = try p <|> P.between Tok.openP Tok.closeP p
  
  manyParensP :: Parser a -> Parser a
  manyParensP p = try p <|> P.between Tok.openP Tok.closeP (manyParensP p)
  
  tablishP :: Parser (Tablish RawNames Range)
  tablishP = do
      table <- singleTableWithViewsP
      joins <- fmap (appEndo . fold . reverse) $ many $ Endo <$> joinP
      return $ joins table
  
  joinP :: Parser (Tablish RawNames Range -> Tablish RawNames Range)
  joinP = do
      maybeJoinType <- optionMaybe $ innerJoinTypeP <|> crossJoinTypeP <|> try semiJoinTypeP <|> outerJoinTypeP
      joinType <- Tok.joinP >>= \ r -> return $ case maybeJoinType of
          Nothing -> JoinInner r
          Just joinType -> (<> r) <$> joinType
  
      rhs <- singleTableWithViewsP
      maybeCondition <- optionMaybe $ do
          _ <- Tok.onP <?> "condition in join clause"
          JoinOn <$> exprP
  
      let condition = case maybeCondition of
              Nothing -> let info = getInfo joinType <> getInfo rhs
                          in JoinOn $ ConstantExpr info $ BooleanConstant info True
              Just c -> c
          joinType' = case (joinType, maybeCondition) of
              -- omitting the ON clause turns a LEFT SEMI JOIN into an INNER JOIN
              (JoinSemi r, Nothing) -> JoinInner r
              _ -> joinType
  
      let r lhs = getInfo lhs <> getInfo rhs <> getInfo condition
      return $ \ lhs ->
          TablishJoin (r lhs) joinType' condition lhs rhs
  
  outerJoinTypeP :: Parser (JoinType Range)
  outerJoinTypeP = do
      joinType <- choice
          [ Tok.leftP >>= \ r -> return $ JoinLeft r
          , Tok.rightP >>= \ r -> return $ JoinRight r
          , Tok.fullP >>= \ r -> return $ JoinFull r
          ]
  
      optional Tok.outerP
  
      return joinType
  
  innerJoinTypeP :: Parser (JoinType Range)
  innerJoinTypeP = Tok.innerP >>= \ r -> return $ JoinInner r
  
  crossJoinTypeP :: Parser (JoinType Range)
  crossJoinTypeP = Tok.crossP >>= \ r -> return $ JoinInner r
  
  semiJoinTypeP :: Parser (JoinType Range)
  semiJoinTypeP = do
    r <- Tok.leftP
    r' <- Tok.semiP
    return $ JoinSemi (r <> r')
  
  constantP :: Parser (Constant Range)
  constantP = choice
      [ uncurry (flip StringConstant)
          <$> (try (optional Tok.timestampP) >> Tok.stringP)
  
      , uncurry (flip NumericConstant) <$> Tok.numberP
      , NullConstant <$> Tok.nullP
      , uncurry (flip BooleanConstant) <$> choice
          [ Tok.trueP >>= \ r -> return (True, r)
          , Tok.falseP >>= \ r -> return (False, r)
          ]
      ]
  