


'''simple docstring'''
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


__magic_name__   :   Dict  							=logging.get_logger(__name__)

__magic_name__   :   int  							={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__magic_name__   :   List[str]  							={
    'vocab_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json',
    },
    'merges_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'openai/clip-vit-base-patch32': (
            'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'
        ),
    },
}

__magic_name__   :   List[Any]  							={
    'openai/clip-vit-base-patch32': 77,
}






class 		UpperCamelCase_		(					A   ):





						"""simple docstring"""




						UpperCAmelCase__	:       Union[str, Any]         =     VOCAB_FILES_NAMES
						UpperCAmelCase__	:       Optional[int]         =     PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase__	:       Optional[Any]         =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase__	:       int         =     ['''input_ids''', '''attention_mask''']
						UpperCAmelCase__	:       List[str]         =     CLIPTokenizer
						def __init__( self				:					Dict	,				_lowerCamelCase				:					Optional[Any]=None	,				_lowerCamelCase				:					Tuple=None	,				_lowerCamelCase				:					Optional[int]=None	,				_lowerCamelCase				:					List[Any]="<|endoftext|>"	,				_lowerCamelCase				:					Optional[int]="<|startoftext|>"	,				_lowerCamelCase				:					Optional[int]="<|endoftext|>"	,				_lowerCamelCase				:					Union[str, Any]="<|endoftext|>"	,				**_lowerCamelCase				:					Union[str, Any]	,				)							->					Optional[Any]:
										super().__init__(
										    _lowerCamelCase	,				_lowerCamelCase	,				tokenizer_file=_lowerCamelCase	,				unk_token=_lowerCamelCase	,				bos_token=_lowerCamelCase	,				eos_token=_lowerCamelCase	,				pad_token=_lowerCamelCase	,				**_lowerCamelCase	,				)

										if not isinstance(self.backend_tokenizer.pre_tokenizer	,				pre_tokenizers.Sequence  ):
														raise ValueError(
														    "The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"
														    " heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"
														    " to be compatible with this version.The easiest way to do so is"
														    " `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"
														    " to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"
														    " transformers."  )

										self._wrap_decode_method_backend_tokenizer()
						def 					__A					( self				:					Dict  )							->					List[Any]:
										__magic_name__	  = self.backend_tokenizer.decode

										def new_decode_method(*_lowerCamelCase				:					Union[str, Any]	,				**_lowerCamelCase				:					Optional[int]  ):
														__magic_name__	  = orig_decode_method(*_lowerCamelCase	,				**_lowerCamelCase  )
														__magic_name__	  = text.replace(self.backend_tokenizer.model.end_of_word_suffix	,				" "  ).strip()
														return text

										__magic_name__	  = new_decode_method
						def 					__A					( self				:					Dict	,				_lowerCamelCase				:					List[int]	,				_lowerCamelCase				:					Optional[List[int]] = None  )							->					List[int]:
										__magic_name__	  = [self.bos_token_id]
										__magic_name__	  = [self.eos_token_id]

										if token_ids_a is None:
														return bos_token + token_ids_a + eos_token
										return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token
						def 					__A					( self				:					str	,				_lowerCamelCase				:					List[int]	,				_lowerCamelCase				:					Optional[List[int]] = None  )							->					List[int]:
										__magic_name__	  = [self.bos_token_id]
										__magic_name__	  = [self.eos_token_id]

										if token_ids_a is None:
														return len(bos_token + token_ids_a + eos_token  ) * [0]
										return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token  ) * [0]

						def 					__A					( self				:					List[str]	,				_lowerCamelCase				:					str	,				_lowerCamelCase				:					Optional[str] = None  )							->					Tuple[str]:
										__magic_name__	  = self._tokenizer.model.save(_lowerCamelCase	,				name=_lowerCamelCase  )
										return tuple(_lowerCamelCase  )







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__   :   List[Any]  							={'configuration_yoso': ['YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP', 'YosoConfig']}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__magic_name__   :   Union[str, Any]  							=[
						    'YOSO_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'YosoForMaskedLM',
						    'YosoForMultipleChoice',
						    'YosoForQuestionAnswering',
						    'YosoForSequenceClassification',
						    'YosoForTokenClassification',
						    'YosoLayer',
						    'YosoModel',
						    'YosoPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_yoso import (
												    YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
												    YosoForMaskedLM,
												    YosoForMultipleChoice,
												    YosoForQuestionAnswering,
												    YosoForSequenceClassification,
												    YosoForTokenClassification,
												    YosoLayer,
												    YosoModel,
												    YosoPreTrainedModel,
												)


else:
						import sys

						__magic_name__   :   Optional[int]  							=_LazyModule(__name__, globals()['__file__'], _import_structure)



