


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCamelCase		      = logging.get_logger(__name__)

lowerCamelCase		      = {'vocab_file': 'vocab.txt'}

lowerCamelCase		      = {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

lowerCamelCase		      = {
    'tau/splinter-base': 5_12,
    'tau/splinter-base-qass': 5_12,
    'tau/splinter-large': 5_12,
    'tau/splinter-large-qass': 5_12,
}

lowerCamelCase		      = {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}




class A						(     UpperCamelCase_  ):
      UpperCamelCase__							:						int   =VOCAB_FILES_NAMES
      UpperCamelCase__							:						Optional[int]   =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__							:						Union[str, Any]   =PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__							:						Optional[int]   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__							:						Union[str, Any]   =SplinterTokenizer






      def __init__(      self		:  Optional[int]    ,     lowercase_		:  str=None    ,     lowercase_		:  Dict=None    ,     lowercase_		:  Dict=True    ,     lowercase_		:  Optional[Any]="[UNK]"    ,     lowercase_		:  Tuple="[SEP]"    ,     lowercase_		:  Tuple="[PAD]"    ,     lowercase_		:  Union[str, Any]="[CLS]"    ,     lowercase_		:  Union[str, Any]="[MASK]"    ,     lowercase_		:  Optional[Any]="[QUESTION]"    ,     lowercase_		:  Dict=True    ,     lowercase_		:  Dict=None    ,     **lowercase_		:  Tuple    ,     )      ->		List[str]:
             """simple docstring"""




             super().__init__(
                 lowercase_    ,     tokenizer_file=lowercase_    ,     do_lower_case=lowercase_    ,     unk_token=lowercase_    ,     sep_token=lowercase_    ,     pad_token=lowercase_    ,     cls_token=lowercase_    ,     mask_token=lowercase_    ,     tokenize_chinese_chars=lowercase_    ,     strip_accents=lowercase_    ,     additional_special_tokens=(question_token,)    ,     **lowercase_    ,     )

             _lowerCamelCase :	List[str]						=json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
             if (
                 pre_tok_state.get('lowercase'    ,     lowercase_				) != do_lower_case
                 or pre_tok_state.get('strip_accents'    ,     lowercase_				) != strip_accents
             ):
                    _lowerCamelCase :	str						=getattr(lowercase_    ,     pre_tok_state.pop('type'				)				)
                    _lowerCamelCase :	Dict						=do_lower_case
                    _lowerCamelCase :	Optional[Any]						=strip_accents
                    _lowerCamelCase :	str						=pre_tok_class(**lowercase_				)

             _lowerCamelCase :	List[str]						=do_lower_case






      @property
      def  lowerCamelCase (      self		:  Dict				)      ->		Tuple:
             """simple docstring"""




             return self.convert_tokens_to_ids(self.question_token				)






      def  lowerCamelCase (      self		:  int    ,     lowercase_		:  List[int]    ,     lowercase_		:  Optional[List[int]] = None				)      ->		List[int]:
             """simple docstring"""




             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

             _lowerCamelCase :	Any						=[self.cls_token_id]
             _lowerCamelCase :	Optional[int]						=[self.sep_token_id]
             _lowerCamelCase :	int						=[self.question_token_id] + [self.convert_tokens_to_ids('.'				)]
             if self.padding_side == "right":
                    # Input is question-then-context
                    return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
             else:
                    # Input is context-then-question
                    return cls + token_ids_a + sep + token_ids_a + question_suffix + sep






      def  lowerCamelCase (      self		:  Any    ,     lowercase_		:  List[int]    ,     lowercase_		:  Optional[List[int]] = None				)      ->		List[int]:
             """simple docstring"""




             _lowerCamelCase :	Tuple						=[self.sep_token_id]
             _lowerCamelCase :	Optional[int]						=[self.cls_token_id]
             _lowerCamelCase :	int						=[self.question_token_id] + [self.convert_tokens_to_ids('.'				)]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep				) * [0]

             if self.padding_side == "right":
                    # Input is question-then-context
                    return len(cls + token_ids_a + question_suffix + sep				) * [0] + len(token_ids_a + sep				) * [1]
             else:
                    # Input is context-then-question
                    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + question_suffix + sep				) * [1]






      def  lowerCamelCase (      self		:  Optional[int]    ,     lowercase_		:  str    ,     lowercase_		:  Optional[str] = None				)      ->		Tuple[str]:
             """simple docstring"""




             _lowerCamelCase :	List[Any]						=self._tokenizer.model.save(lowercase_    ,     name=lowercase_				)
             return tuple(lowercase_				)






from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCamelCase		      = logging.get_logger(__name__)

lowerCamelCase		      = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase		      = {
    'vocab_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json',
    },
    'merges_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'openai/clip-vit-base-patch32': (
            'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase		      = {
    'openai/clip-vit-base-patch32': 77,
}




class A						(     UpperCamelCase_  ):
      UpperCamelCase__							:						Dict   =VOCAB_FILES_NAMES
      UpperCamelCase__							:						Optional[int]   =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__							:						Optional[int]   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__							:						Optional[Any]   =['input_ids', 'attention_mask']
      UpperCamelCase__							:						str   =CLIPTokenizer






      def __init__(      self		:  str    ,     lowercase_		:  Optional[Any]=None    ,     lowercase_		:  Optional[int]=None    ,     lowercase_		:  int=None    ,     lowercase_		:  Optional[int]="<|endoftext|>"    ,     lowercase_		:  Union[str, Any]="<|startoftext|>"    ,     lowercase_		:  Tuple="<|endoftext|>"    ,     lowercase_		:  int="<|endoftext|>"    ,     **lowercase_		:  Optional[Any]    ,     )      ->		Any:
             """simple docstring"""




             super().__init__(
                 lowercase_    ,     lowercase_    ,     tokenizer_file=lowercase_    ,     unk_token=lowercase_    ,     bos_token=lowercase_    ,     eos_token=lowercase_    ,     pad_token=lowercase_    ,     **lowercase_    ,     )

             if not isinstance(self.backend_tokenizer.pre_tokenizer    ,     pre_tokenizers.Sequence				):
                    raise ValueError(
                        'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
                        ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
                        ' to be compatible with this version.The easiest way to do so is'
                        ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
                        ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
                        ' transformers.'				)

             self._wrap_decode_method_backend_tokenizer()






      def  lowerCamelCase (      self		:  Any				)      ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase :	int						=self.backend_tokenizer.decode

             def new_decode_method(*lowercase_		:  List[str]    ,     **lowercase_		:  int				):
                    _lowerCamelCase :	Tuple						=orig_decode_method(*lowercase_    ,     **lowercase_				)
                    _lowerCamelCase :	Dict						=text.replace(self.backend_tokenizer.model.end_of_word_suffix    ,     ' '				).strip()
                    return text

             _lowerCamelCase :	List[Any]						=new_decode_method






      def  lowerCamelCase (      self		:  Dict    ,     lowercase_		:  List[int]    ,     lowercase_		:  Optional[List[int]] = None				)      ->		List[int]:
             """simple docstring"""




             _lowerCamelCase :	Union[str, Any]						=[self.bos_token_id]
             _lowerCamelCase :	str						=[self.eos_token_id]

             if token_ids_a is None:
                    return bos_token + token_ids_a + eos_token
             return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token






      def  lowerCamelCase (      self		:  Any    ,     lowercase_		:  List[int]    ,     lowercase_		:  Optional[List[int]] = None				)      ->		List[int]:
             """simple docstring"""




             _lowerCamelCase :	int						=[self.bos_token_id]
             _lowerCamelCase :	Any						=[self.eos_token_id]

             if token_ids_a is None:
                    return len(bos_token + token_ids_a + eos_token				) * [0]
             return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token				) * [0]






      def  lowerCamelCase (      self		:  str    ,     lowercase_		:  str    ,     lowercase_		:  Optional[str] = None				)      ->		Tuple[str]:
             """simple docstring"""




             _lowerCamelCase :	str						=self._tokenizer.model.save(lowercase_    ,     name=lowercase_				)
             return tuple(lowercase_				)


