


"""simple docstring"""


from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


__SCREAMING_SNAKE_CASE      =      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      =      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE      =      {
    'vocab_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json',
    },
    'merges_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'openai/clip-vit-base-patch32': (
            'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'
        ),
    },
}

__SCREAMING_SNAKE_CASE      =      {
    'openai/clip-vit-base-patch32': 77,
}




class 	__UpperCamelCase    (    lowercase_	):
					lowercase_						:							str			=     VOCAB_FILES_NAMES
					lowercase_						:							List[Any]			=     PRETRAINED_VOCAB_FILES_MAP
					lowercase_						:							Optional[int]			=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_						:							Dict			=     ["""input_ids""", """attention_mask"""]
					lowercase_						:							int			=     CLIPTokenizer
					def __init__(       self       :  Any    ,   UpperCAmelCase       :  Dict=None    ,   UpperCAmelCase       :  List[str]=None    ,   UpperCAmelCase       :  str=None    ,   UpperCAmelCase       :  Any="<|endoftext|>"    ,   UpperCAmelCase       :  int="<|startoftext|>"    ,   UpperCAmelCase       :  Dict="<|endoftext|>"    ,   UpperCAmelCase       :  Any="<|endoftext|>"    ,   **UpperCAmelCase       :  List[str]    ,   )  ->					Optional[int]:
											super().__init__(
											    UpperCAmelCase    ,   UpperCAmelCase    ,   tokenizer_file=UpperCAmelCase    ,   unk_token=UpperCAmelCase    ,   bos_token=UpperCAmelCase    ,   eos_token=UpperCAmelCase    ,   pad_token=UpperCAmelCase    ,   **UpperCAmelCase    ,   )

											if not isinstance(self.backend_tokenizer.pre_tokenizer    ,   pre_tokenizers.Sequence      ):
																	raise ValueError(
																	    'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
																	    ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
																	    ' to be compatible with this version.The easiest way to do so is'
																	    ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
																	    ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
																	    ' transformers.'      )

											self._wrap_decode_method_backend_tokenizer()
					def 					UpperCAmelCase__   (       self       :  int      )  ->					str:
											lowerCAmelCase      :Any				  = self.backend_tokenizer.decode

											def new_decode_method(*UpperCAmelCase       :  Optional[int]    ,   **UpperCAmelCase       :  List[Any]      ):
																	lowerCAmelCase      :Union[str, Any]				  = orig_decode_method(*UpperCAmelCase    ,   **UpperCAmelCase      )
																	lowerCAmelCase      :Tuple				  = text.replace(self.backend_tokenizer.model.end_of_word_suffix    ,   ' '      ).strip()
																	return text

											lowerCAmelCase      :str				  = new_decode_method
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]    ,   UpperCAmelCase       :  List[int]    ,   UpperCAmelCase       :  Optional[List[int]] = None      )  ->					Optional[int]:
											lowerCAmelCase      :Any				  = [self.bos_token_id]
											lowerCAmelCase      :Union[str, Any]				  = [self.eos_token_id]

											if token_ids_a is None:
																	return bos_token + token_ids_a + eos_token
											return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  List[int]    ,   UpperCAmelCase       :  Optional[List[int]] = None      )  ->					str:
											lowerCAmelCase      :Optional[Any]				  = [self.bos_token_id]
											lowerCAmelCase      :Optional[Any]				  = [self.eos_token_id]

											if token_ids_a is None:
																	return len(bos_token + token_ids_a + eos_token      ) * [0]
											return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token      ) * [0]
					def 					UpperCAmelCase__   (       self       :  int    ,   UpperCAmelCase       :  str    ,   UpperCAmelCase       :  Optional[str] = None      )  ->					Union[str, Any]:
											lowerCAmelCase      :Tuple				  = self._tokenizer.model.save(UpperCAmelCase    ,   name=UpperCAmelCase      )
											return tuple(UpperCAmelCase      )







"""simple docstring"""


import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir




@dataclass
class 	__UpperCamelCase    :
					lowercase_						:							str




class 	__UpperCamelCase    (    unittest.TestCase	):
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[str]				  = get_tests_dir('fixtures/test_sentencepiece.model'      )
											lowerCAmelCase      :Tuple				  = get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'      )

											lowerCAmelCase      :List[str]				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :Dict				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   0      )

											lowerCAmelCase      :int				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :List[str]				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   1      )

											self.assertIn(
											    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
											    ' which is not implemented in the fast tokenizers.'    ,   str(w[0].message      )    ,   )