





"""simple docstring"""
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
   import sqlitea

   import sqlalchemy


lowerCAmelCase:	Tuple							      =datasets.utils.logging.get_logger(__name__)
@dataclass
class lowerCamelCase__  (    datasets.BuilderConfig				):
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					True
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					None
   __UpperCAmelCase        =					10000
   __UpperCAmelCase        =					None



   def    _UpperCAmelCase	(       self	) ->    str:







      """simple docstring"""
      if self.sql is None:
         raise ValueError("""sql must be specified"""	)
      if self.con is None:
         raise ValueError("""con must be specified"""	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case = None      ,   ) ->    str:







      """simple docstring"""
      lowercase    :			Union[str, Any]  =       config_kwargs.copy()
      # We need to stringify the Selectable object to make its hash deterministic

      # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
      lowercase    :			Union[str, Any]  =       config_kwargs["""sql"""]
      if not isinstance(snake_case      ,   snake_case	):
         if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
            import sqlalchemy

            if isinstance(snake_case      ,   sqlalchemy.sql.Selectable	):
               lowercase    :			List[str]  =       sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""	)[0] + """://"""	)
               lowercase    :			Optional[Any]  =       str(sql.compile(dialect=engine.dialect	)	)
               lowercase    :			Union[str, Any]  =       sql_str
            else:
               raise TypeError(
                   f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case	)}: {sql}'''	)
         else:
            raise TypeError(
                f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case	)}: {sql}'''	)
      lowercase    :			Union[str, Any]  =       config_kwargs["""con"""]
      if not isinstance(snake_case      ,   snake_case	):
         lowercase    :			List[str]  =       id(snake_case	)
         logger.info(
             f'''SQL connection \'con\' of type {type(snake_case	)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''	)

      return super().create_config_id(snake_case      ,   custom_features=snake_case	)





   @property
   def    _UpperCAmelCase	(       self	) ->    Optional[int]:







      """simple docstring"""
      lowercase    :			Tuple  =       {
          """index_col""": self.index_col,
          """columns""": self.columns,
          """params""": self.params,
          """coerce_float""": self.coerce_float,
          """parse_dates""": self.parse_dates,
      }
      return pd_read_sql_kwargs




class lowerCamelCase__  (    datasets.ArrowBasedBuilder				):
   __UpperCAmelCase        =					SqlConfig



   def    _UpperCAmelCase	(       self	) ->    str:







      """simple docstring"""
      return datasets.DatasetInfo(features=self.config.features	)



   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    int:







      """simple docstring"""
      return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,   gen_kwargs={}	)]



   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    pa.Table:







      """simple docstring"""
      if self.config.features is not None:
         lowercase    :			Optional[int]  =       self.config.features.arrow_schema
         if all(not require_storage_cast(snake_case	) for feature in self.config.features.values()	):
            # cheaper cast
            lowercase    :			str  =       pa.Table.from_arrays([pa_table[field.name] for field in schema]      ,   schema=snake_case	)
         else:
            # more expensive cast; allows str <-> int/float or str to Audio for example
            lowercase    :			int  =       table_cast(snake_case      ,   snake_case	)
      return pa_table





   def    _UpperCAmelCase	(       self	) ->    str:







      """simple docstring"""
      lowercase    :			int  =       self.config.chunksize
      lowercase    :			Dict  =       pd.read_sql(
          self.config.sql      ,   self.config.con      ,   chunksize=snake_case      ,   **self.config.pd_read_sql_kwargs	)
      lowercase    :			str  =       [sql_reader] if chunksize is None else sql_reader
      for chunk_idx, df in enumerate(snake_case	):
         lowercase    :			Optional[Any]  =       pa.Table.from_pandas(snake_case	)
         yield chunk_idx, self._cast_table(snake_case	)







"""simple docstring"""
from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class lowerCamelCase__  (    __UpperCamelCase				):
   __UpperCAmelCase        =					"""MarkupLMFeatureExtractor"""
   __UpperCAmelCase        =					("""MarkupLMTokenizer""", """MarkupLMTokenizerFast""")
   __UpperCAmelCase        =					True



   def __call__(       self      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case = True      ,   snake_case = False      ,   snake_case = None      ,   snake_case = None      ,   snake_case = 0      ,   snake_case = None      ,   snake_case = None      ,   snake_case = None      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = True      ,   snake_case = None      ,   **snake_case      ,   ) ->    BatchEncoding:







      """simple docstring"""
      # first, create nodes and xpaths
      if self.parse_html:
         if html_strings is None:
            raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""	)

         if nodes is not None or xpaths is not None or node_labels is not None:
            raise ValueError(
                """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""	)

         lowercase    :			List[Any]  =       self.feature_extractor(snake_case	)
         lowercase    :			Union[str, Any]  =       features["""nodes"""]
         lowercase    :			List[Any]  =       features["""xpaths"""]
      else:
         if html_strings is not None:
            raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""	)
         if nodes is None or xpaths is None:
            raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""	)

        # # second, apply the tokenizer
      if questions is not None and self.parse_html:
         if isinstance(snake_case      ,   snake_case	):
            lowercase    :			Dict  =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

      lowercase    :			Dict  =       self.tokenizer(
          text=questions if questions is not None else nodes      ,   text_pair=nodes if questions is not None else None      ,   xpaths=snake_case      ,   node_labels=snake_case      ,   add_special_tokens=snake_case      ,   padding=snake_case      ,   truncation=snake_case      ,   max_length=snake_case      ,   stride=snake_case      ,   pad_to_multiple_of=snake_case      ,   return_token_type_ids=snake_case      ,   return_attention_mask=snake_case      ,   return_overflowing_tokens=snake_case      ,   return_special_tokens_mask=snake_case      ,   return_offsets_mapping=snake_case      ,   return_length=snake_case      ,   verbose=snake_case      ,   return_tensors=snake_case      ,   **snake_case      ,   )

      return encoded_inputs



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Optional[Any]:







      """simple docstring"""
      return self.tokenizer.batch_decode(*snake_case      ,   **snake_case	)



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Tuple:







      """simple docstring"""
      return self.tokenizer.decode(*snake_case      ,   **snake_case	)





   @property
   def    _UpperCAmelCase	(       self	) ->    Dict:







      """simple docstring"""
      lowercase    :			Any  =       self.tokenizer.model_input_names
      return tokenizer_input_names


