
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


A__       :   Dict   						=					logging.get_logger(__name__)

A__       :   Union[str, Any]   						=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A__       :   List[str]   						=					{
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

A__       :   Optional[int]   						=					{
    """openai/clip-vit-base-patch32""": 7_7,
}





class 						lowercase       (  __UpperCamelCase					):
     __a					=    VOCAB_FILES_NAMES
     __a					=    PRETRAINED_VOCAB_FILES_MAP
     __a					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __a					=    ["""input_ids""", """attention_mask"""]
     __a					=    CLIPTokenizer
     def __init__(		self ,	SCREAMING_SNAKE_CASE__=None ,	SCREAMING_SNAKE_CASE__=None ,	SCREAMING_SNAKE_CASE__=None ,	SCREAMING_SNAKE_CASE__="<|endoftext|>" ,	SCREAMING_SNAKE_CASE__="<|startoftext|>" ,	SCREAMING_SNAKE_CASE__="<|endoftext|>" ,	SCREAMING_SNAKE_CASE__="<|endoftext|>" ,	**SCREAMING_SNAKE_CASE__ ,	):
        """simple docstring"""







        super().__init__(
            SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	tokenizer_file=SCREAMING_SNAKE_CASE__ ,	unk_token=SCREAMING_SNAKE_CASE__ ,	bos_token=SCREAMING_SNAKE_CASE__ ,	eos_token=SCREAMING_SNAKE_CASE__ ,	pad_token=SCREAMING_SNAKE_CASE__ ,	**SCREAMING_SNAKE_CASE__ ,	)

        if not isinstance(self.backend_tokenizer.pre_tokenizer ,	pre_tokenizers.Sequence    ):
           raise ValueError(
               '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
               ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
               ''' to be compatible with this version.The easiest way to do so is'''
               ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
               ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
               ''' transformers.'''    )

        self._wrap_decode_method_backend_tokenizer()
     def lowercase_			(		self    ):
        """simple docstring"""







        lowerCAmelCase__ :    List[Any]				      =			self.backend_tokenizer.decode

        def new_decode_method(*SCREAMING_SNAKE_CASE__ ,	**SCREAMING_SNAKE_CASE__    ):
           lowerCAmelCase__ :    Optional[int]				      =			orig_decode_method(*SCREAMING_SNAKE_CASE__ ,	**SCREAMING_SNAKE_CASE__    )
           lowerCAmelCase__ :    Any				      =			text.replace(self.backend_tokenizer.model.end_of_word_suffix ,	''' '''    ).strip()
           return text

        lowerCAmelCase__ :    str				      =			new_decode_method
     def lowercase_			(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ = None    ):
        """simple docstring"""







        lowerCAmelCase__ :    int				      =			[self.bos_token_id]
        lowerCAmelCase__ :    Optional[int]				      =			[self.eos_token_id]

        if token_ids_a is None:
           return bos_token + token_ids_a + eos_token
        return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token
     def lowercase_			(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ = None    ):
        """simple docstring"""







        lowerCAmelCase__ :    Union[str, Any]				      =			[self.bos_token_id]
        lowerCAmelCase__ :    Any				      =			[self.eos_token_id]

        if token_ids_a is None:
           return len(bos_token + token_ids_a + eos_token    ) * [0]
        return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token    ) * [0]
     def lowercase_			(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ = None    ):
        """simple docstring"""







        lowerCAmelCase__ :    List[str]				      =			self._tokenizer.model.save(SCREAMING_SNAKE_CASE__ ,	name=SCREAMING_SNAKE_CASE__    )
        return tuple(SCREAMING_SNAKE_CASE__    )





# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
A__       :   Optional[Any]   						=					abspath(join(dirname(dirname(dirname(__file__))), """src"""))
sys.path.insert(1, git_repo_path)


# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)



def      _a     (   __UpperCamelCase       :  Tuple    ):
			from transformers.testing_utils import pytest_addoption_shared

			pytest_addoption_shared(__UpperCamelCase    )



def      _a     (   __UpperCamelCase       :  int    ):
			from transformers.testing_utils import pytest_terminal_summary_main

			lowerCAmelCase__ :    int				      =			terminalreporter.config.getoption('''--make-reports'''    )
			if make_reports:
						pytest_terminal_summary_main(__UpperCamelCase	,id=__UpperCamelCase    )



