





"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase:	Dict							      =logging.get_logger(__name__)

lowerCAmelCase:	Optional[int]							      ={"""vocab_file""": """vocab.txt"""}

lowerCAmelCase:	Dict							      ={
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

lowerCAmelCase:	Dict							      ={
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

lowerCAmelCase:	Union[str, Any]							      ={
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}
class lowerCamelCase__  (    _a				):
   __UpperCAmelCase        =					VOCAB_FILES_NAMES
   __UpperCAmelCase        =					PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase        =					PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase        =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase        =					SplinterTokenizer



   def __init__(       self      ,   snake_case=None      ,   snake_case=None      ,   snake_case=True      ,   snake_case="[UNK]"      ,   snake_case="[SEP]"      ,   snake_case="[PAD]"      ,   snake_case="[CLS]"      ,   snake_case="[MASK]"      ,   snake_case="[QUESTION]"      ,   snake_case=True      ,   snake_case=None      ,   **snake_case      ,   ) ->    Dict:







      """simple docstring"""
      super().__init__(
          lowerCamelCase__      ,   tokenizer_file=lowerCamelCase__      ,   do_lower_case=lowerCamelCase__      ,   unk_token=lowerCamelCase__      ,   sep_token=lowerCamelCase__      ,   pad_token=lowerCamelCase__      ,   cls_token=lowerCamelCase__      ,   mask_token=lowerCamelCase__      ,   tokenize_chinese_chars=lowerCamelCase__      ,   strip_accents=lowerCamelCase__      ,   additional_special_tokens=(question_token,)      ,   **lowerCamelCase__      ,   )

      lowercase    :			Union[str, Any]  =       json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
      if (
          pre_tok_state.get("""lowercase"""      ,   lowerCamelCase__	) != do_lower_case
          or pre_tok_state.get("""strip_accents"""      ,   lowerCamelCase__	) != strip_accents
      ):
         lowercase    :			Optional[Any]  =       getattr(lowerCamelCase__      ,   pre_tok_state.pop("""type"""	)	)
         lowercase    :			List[Any]  =       do_lower_case
         lowercase    :			str  =       strip_accents
         lowercase    :			int  =       pre_tok_class(**lowerCamelCase__	)

      lowercase    :			Dict  =       do_lower_case



   @property
   def    _UpperCAmelCase	(       self	) ->    List[Any]:







      """simple docstring"""
      return self.convert_tokens_to_ids(self.question_token	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case = None	) ->    Optional[int]:







      """simple docstring"""
      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      lowercase    :			Any  =       [self.cls_token_id]
      lowercase    :			Tuple  =       [self.sep_token_id]
      lowercase    :			List[Any]  =       [self.question_token_id] + [self.convert_tokens_to_ids("""."""	)]
      if self.padding_side == "right":
         # Input is question-then-context
         return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
      else:
         # Input is context-then-question
         return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case = None	) ->    List[str]:







      """simple docstring"""
      lowercase    :			int  =       [self.sep_token_id]
      lowercase    :			List[Any]  =       [self.cls_token_id]
      lowercase    :			List[Any]  =       [self.question_token_id] + [self.convert_tokens_to_ids("""."""	)]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]

      if self.padding_side == "right":
         # Input is question-then-context
         return len(cls + token_ids_a + question_suffix + sep	) * [0] + len(token_ids_a + sep	) * [1]
      else:
         # Input is context-then-question
         return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + question_suffix + sep	) * [1]





   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case = None	) ->    Any:







      """simple docstring"""
      lowercase    :			List[Any]  =       self._tokenizer.model.save(lowerCamelCase__      ,   name=lowerCamelCase__	)
      return tuple(lowerCamelCase__	)







"""simple docstring"""
from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class lowerCamelCase__  (    __UpperCamelCase				):
   __UpperCAmelCase        =					"""MarkupLMFeatureExtractor"""
   __UpperCAmelCase        =					("""MarkupLMTokenizer""", """MarkupLMTokenizerFast""")
   __UpperCAmelCase        =					True



   def __call__(       self      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case = True      ,   snake_case = False      ,   snake_case = None      ,   snake_case = None      ,   snake_case = 0      ,   snake_case = None      ,   snake_case = None      ,   snake_case = None      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = True      ,   snake_case = None      ,   **snake_case      ,   ) ->    BatchEncoding:







      """simple docstring"""
      # first, create nodes and xpaths
      if self.parse_html:
         if html_strings is None:
            raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""	)

         if nodes is not None or xpaths is not None or node_labels is not None:
            raise ValueError(
                """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""	)

         lowercase    :			List[Any]  =       self.feature_extractor(snake_case	)
         lowercase    :			Union[str, Any]  =       features["""nodes"""]
         lowercase    :			List[Any]  =       features["""xpaths"""]
      else:
         if html_strings is not None:
            raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""	)
         if nodes is None or xpaths is None:
            raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""	)

        # # second, apply the tokenizer
      if questions is not None and self.parse_html:
         if isinstance(snake_case      ,   snake_case	):
            lowercase    :			Dict  =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

      lowercase    :			Dict  =       self.tokenizer(
          text=questions if questions is not None else nodes      ,   text_pair=nodes if questions is not None else None      ,   xpaths=snake_case      ,   node_labels=snake_case      ,   add_special_tokens=snake_case      ,   padding=snake_case      ,   truncation=snake_case      ,   max_length=snake_case      ,   stride=snake_case      ,   pad_to_multiple_of=snake_case      ,   return_token_type_ids=snake_case      ,   return_attention_mask=snake_case      ,   return_overflowing_tokens=snake_case      ,   return_special_tokens_mask=snake_case      ,   return_offsets_mapping=snake_case      ,   return_length=snake_case      ,   verbose=snake_case      ,   return_tensors=snake_case      ,   **snake_case      ,   )

      return encoded_inputs



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Optional[Any]:







      """simple docstring"""
      return self.tokenizer.batch_decode(*snake_case      ,   **snake_case	)



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Tuple:







      """simple docstring"""
      return self.tokenizer.decode(*snake_case      ,   **snake_case	)





   @property
   def    _UpperCAmelCase	(       self	) ->    Dict:







      """simple docstring"""
      lowercase    :			Any  =       self.tokenizer.model_input_names
      return tokenizer_input_names


