


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCamelCase__							:List[Any]       =  logging.get_logger(__name__)

UpperCamelCase__							:Any       =  {"""vocab_file""": """vocab.txt"""}

UpperCamelCase__							:List[str]       =  {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

UpperCamelCase__							:int       =  {
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

UpperCamelCase__							:List[str]       =  {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}






class     A(     lowerCamelCase__		):

		"""simple docstring"""





		A         =       VOCAB_FILES_NAMES
		A         =       PRETRAINED_VOCAB_FILES_MAP
		A         =       PRETRAINED_INIT_CONFIGURATION
		A         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A         =       SplinterTokenizer
		def __init__(   self				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=True				,							SCREAMING_SNAKE_CASE__="[UNK]"				,							SCREAMING_SNAKE_CASE__="[SEP]"				,							SCREAMING_SNAKE_CASE__="[PAD]"				,							SCREAMING_SNAKE_CASE__="[CLS]"				,							SCREAMING_SNAKE_CASE__="[MASK]"				,							SCREAMING_SNAKE_CASE__="[QUESTION]"				,							SCREAMING_SNAKE_CASE__=True				,							SCREAMING_SNAKE_CASE__=None				,							**SCREAMING_SNAKE_CASE__				,							)			->  Optional[int]:

					"""simple docstring"""



					super().__init__(
					    SCREAMING_SNAKE_CASE__				,							tokenizer_file=SCREAMING_SNAKE_CASE__				,							do_lower_case=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							tokenize_chinese_chars=SCREAMING_SNAKE_CASE__				,							strip_accents=SCREAMING_SNAKE_CASE__				,							additional_special_tokens=(question_token,)				,							**SCREAMING_SNAKE_CASE__				,							)

					_UpperCamelCase				:List[str]				    =  json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
					if (
					    pre_tok_state.get('''lowercase'''				,							SCREAMING_SNAKE_CASE__      ) != do_lower_case
					    or pre_tok_state.get('''strip_accents'''				,							SCREAMING_SNAKE_CASE__      ) != strip_accents
					):
								_UpperCamelCase				:Optional[Any]				    =  getattr(SCREAMING_SNAKE_CASE__				,							pre_tok_state.pop('''type'''      )      )
								_UpperCamelCase				:Dict				    =  do_lower_case
								_UpperCamelCase				:List[Any]				    =  strip_accents
								_UpperCamelCase				:List[str]				    =  pre_tok_class(**SCREAMING_SNAKE_CASE__      )

					_UpperCamelCase				:Tuple				    =  do_lower_case
		@property
		def      _UpperCamelCase(   self      )			->  List[Any]:

					"""simple docstring"""



					return self.convert_tokens_to_ids(self.question_token      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  List[int]:

					"""simple docstring"""



					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					_UpperCamelCase				:str				    =  [self.cls_token_id]
					_UpperCamelCase				:Any				    =  [self.sep_token_id]
					_UpperCamelCase				:List[str]				    =  [self.question_token_id] + [self.convert_tokens_to_ids('''.'''      )]
					if self.padding_side == "right":
								# Input is question-then-context
								return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
					else:
								# Input is context-then-question
								return cls + token_ids_a + sep + token_ids_a + question_suffix + sep
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  List[int]:

					"""simple docstring"""



					_UpperCamelCase				:Tuple				    =  [self.sep_token_id]
					_UpperCamelCase				:Any				    =  [self.cls_token_id]
					_UpperCamelCase				:Optional[int]				    =  [self.question_token_id] + [self.convert_tokens_to_ids('''.'''      )]
					if token_ids_a is None:
								return len(cls + token_ids_a + sep      ) * [0]

					if self.padding_side == "right":
								# Input is question-then-context
								return len(cls + token_ids_a + question_suffix + sep      ) * [0] + len(token_ids_a + sep      ) * [1]
					else:
								# Input is context-then-question
								return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + question_suffix + sep      ) * [1]





		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  Tuple[str]:

					"""simple docstring"""



					_UpperCamelCase				:List[Any]				    =  self._tokenizer.model.save(SCREAMING_SNAKE_CASE__				,							name=SCREAMING_SNAKE_CASE__      )
					return tuple(SCREAMING_SNAKE_CASE__      )





"""simple docstring"""






class     A:

		"""simple docstring"""





		def __init__(   self      )			->  List[str]:

					"""simple docstring"""



					_UpperCamelCase				:List[Any]				    =  []
					_UpperCamelCase				:Dict				    =  0
					_UpperCamelCase				:List[Any]				    =  0
		def __str__(   self      )			->  Optional[int]:

					"""simple docstring"""



					_UpperCamelCase				:Optional[Any]				    =  '''<''' + str(self.entries      )[1:-1] + '''>'''
					return printed
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  Tuple:

					"""simple docstring"""



					self.entries.append(SCREAMING_SNAKE_CASE__      )
					_UpperCamelCase				:Union[str, Any]				    =  self.length + 1
		def      _UpperCamelCase(   self      )			->  Dict:

					"""simple docstring"""



					_UpperCamelCase				:str				    =  self.length - 1
					_UpperCamelCase				:Optional[int]				    =  self.entries[self.front]
					# self.front-=1
					# self.entries = self.entries[self.front:]
					_UpperCamelCase				:Tuple				    =  self.entries[1:]
					return dequeued
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  str:

					"""simple docstring"""



					for _ in range(SCREAMING_SNAKE_CASE__      ):
								self.put(self.get()      )
		def      _UpperCamelCase(   self      )			->  str:

					"""simple docstring"""



					return self.entries[0]
		def      _UpperCamelCase(   self      )			->  Union[str, Any]:

					"""simple docstring"""



					return self.length

