




'''simple docstring'''





import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


UpperCamelCase_		:    Any									=		{}
for task, _ in pipeline_test_mapping.items():
   UpperCamelCase_		:    Union[str, Any]									=		{'''pt''': None, '''tf''': None}


# DO **NOT** add item to this set (unless the reason is approved)
UpperCamelCase_		:    List[str]									=		{
    '''tests/models/esm/test_modeling_esmfold.py''',  # The pipeline test mapping is added to `test_modeling_esm.py`
}





def 			__a			(     _UpperCamelCase:					int					)		->				Dict:

     """simple docstring"""




     if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
          return "pt"
     elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
          return "tf"
     elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
          return "flax"
     else:
          return None





def 			__a			(     _UpperCamelCase:					Optional[Any]      ,		_UpperCamelCase:					str					)		->				Union[str, Any]:

     """simple docstring"""




     if PIPELINE_TEST_MAPPING[task].get(_UpperCamelCase      ,		_UpperCamelCase					) is not None:
          return PIPELINE_TEST_MAPPING[task][framework]

     _snake_case								=   pipeline_test_mapping[task]["test"]
     _snake_case								=   None

     if framework == "pt":
          _snake_case								=   getattr(_UpperCamelCase      ,		"model_mapping"      ,		_UpperCamelCase					)
     elif framework == "tf":
          _snake_case								=   getattr(_UpperCamelCase      ,		"tf_model_mapping"      ,		_UpperCamelCase					)

     if mapping is not None:
          _snake_case								=   dict(mapping.items()					)

     # cache the results
     _snake_case								=   mapping
     return mapping





def 			__a			(     _UpperCamelCase:					int      ,		_UpperCamelCase:					List[Any]					)		->				str:

     """simple docstring"""




     _snake_case								=   get_framework(_UpperCamelCase					)
     if framework is None:
          return None
     _snake_case								=   get_mapping_for_task(_UpperCamelCase      ,		_UpperCamelCase					)
     if mapping is None:
          return None

     _snake_case								=   list({model_class.config_class for model_class in test_class.all_model_classes}					)
     if len(_UpperCamelCase					) != 1:
          raise ValueError("There should be exactly one configuration class from `test_class.all_model_classes`."					)

     # This could be a list/tuple of model classes, but it's rare.
     _snake_case								=   mapping.get(config_classes[0]      ,		_UpperCamelCase					)
     if isinstance(_UpperCamelCase      ,		(tuple, list)					):
          _snake_case								=   sorted(_UpperCamelCase      ,		key=lambda _UpperCamelCase					: x.__name__					)

     return model_class





def 			__a			(     _UpperCamelCase:					List[Any]					)		->				Any:

     """simple docstring"""




     _snake_case								=   [(task, get_model_for_pipeline_test(_UpperCamelCase      ,		_UpperCamelCase					)) for task in pipeline_test_mapping]
     _snake_case								=   sorted([(task, model) for task, model in mapping if model is not None]      ,		key=lambda _UpperCamelCase					: x[0]					)

     return dict(_UpperCamelCase					)





def 			__a			(     _UpperCamelCase:					Optional[int]					)		->				Union[str, Any]:

     """simple docstring"""




     _snake_case								=   get_framework(_UpperCamelCase					)
     if framework == "pt":
          _snake_case								=   "torch"
     _snake_case								=   "{}"

     _snake_case								=   get_pipeline_model_mapping(_UpperCamelCase					)
     if len(_UpperCamelCase					) == 0:
          return ""

     _snake_case								=   []
     for task, model_classes in mapping.items():
          if isinstance(_UpperCamelCase      ,		(tuple, list)					):
               # A list/tuple of model classes
               _snake_case								=   "(" + ", ".join([x.__name__ for x in model_classes]					) + ")"
          else:
               # A single model class
               _snake_case								=   model_classes.__name__
          texts.append(F"""\"{task}\": {value}"""					)
     _snake_case								=   "{" + ", ".join(_UpperCamelCase					) + "}"
     _snake_case								=   F"""pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}"""

     return text





def 			__a			(     _UpperCamelCase:					Union[str, Any]					)		->				str:

     """simple docstring"""




     _snake_case								=   {"ModelTesterMixin", "TFModelTesterMixin", "FlaxModelTesterMixin"}
     if not issubclass(_UpperCamelCase      ,		unittest.TestCase					):
          return False
     return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]					)					) > 0





def 			__a			(     _UpperCamelCase:					str					)		->				List[Any]:

     """simple docstring"""




     _snake_case								=   [x for x in get_test_classes(_UpperCamelCase					) if is_valid_test_class(_UpperCamelCase					)]

     _snake_case								=   None
     for test_class in test_classes:
          # If a test class has defined `pipeline_model_mapping`, let's take it
          if getattr(_UpperCamelCase      ,		"pipeline_model_mapping"      ,		_UpperCamelCase					) is not None:
               _snake_case								=   test_class
               break
    # Take the test class with the shortest name (just a heuristic)
     if target_test_class is None and len(_UpperCamelCase					) > 0:
          _snake_case								=   sorted(_UpperCamelCase      ,		key=lambda _UpperCamelCase					: (len(x.__name__					), x.__name__)					)[0]

     return target_test_class





def 			__a			(     _UpperCamelCase:					int      ,		_UpperCamelCase:					Any      ,		_UpperCamelCase:					List[Any]					)		->				Optional[Any]:

     """simple docstring"""




     _snake_case								=   start_idx
     for idx, line in enumerate(lines[start_idx:]					):
          _snake_case								=   len(_UpperCamelCase					) - len(line.lstrip()					)
          if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
               _snake_case								=   start_idx + idx
          elif idx > 0 and indent <= indent_level:
               # Outside the definition block of `pipeline_model_mapping`
               break

     return end_idx





def 			__a			(     _UpperCamelCase:					Dict      ,		_UpperCamelCase:					Any=False					)		->				List[Any]:

     """simple docstring"""




     if getattr(_UpperCamelCase      ,		"pipeline_model_mapping"      ,		_UpperCamelCase					) is not None:
          if not overwrite:
               return "", -1

     _snake_case								=   get_pipeline_model_mapping_string(_UpperCamelCase					)
     if len(_UpperCamelCase					) == 0:
          return "", -1
     _snake_case								=   line_to_add + "\n"

     # The code defined the class `test_class`
     _snake_case							,      _snake_case								=   inspect.getsourcelines(_UpperCamelCase					)
     # `inspect` gives the code for an object, including decorator(s) if any.
     # We (only) need the exact line of the class definition.
     for idx, line in enumerate(_UpperCamelCase					):
          if line.lstrip().startswith("class "					):
               _snake_case								=   class_lines[idx:]
               class_start_line_no += idx
               break
     _snake_case								=   class_start_line_no + len(_UpperCamelCase					) - 1

     # The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
     # `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
     # block that appears in a `test_class`.
     _snake_case								=   None
     # The indent level of the line at `class_lines[start_idx]` (if defined)
     _snake_case								=   0
     # To record if `pipeline_model_mapping` is found in `test_class`.
     _snake_case								=   None
     for idx, line in enumerate(_UpperCamelCase					):
          if line.strip().startswith("all_model_classes = "					):
               _snake_case								=   len(_UpperCamelCase					) - len(line.lstrip()					)
               _snake_case								=   idx
          elif line.strip().startswith("all_generative_model_classes = "					):
               _snake_case								=   len(_UpperCamelCase					) - len(line.lstrip()					)
               _snake_case								=   idx
          elif line.strip().startswith("pipeline_model_mapping = "					):
               _snake_case								=   len(_UpperCamelCase					) - len(line.lstrip()					)
               _snake_case								=   idx
               _snake_case								=   line
               break

     if start_idx is None:
          return "", -1
     # Find the ending index (inclusive) of the above found block.
     _snake_case								=   find_block_ending(_UpperCamelCase      ,		_UpperCamelCase      ,		_UpperCamelCase					)

     # Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
     # `is_timm_available()` instead of `is_torch_available()`.
     # Keep leading and trailing whitespaces
     _snake_case								=   re.compile(r"\s(is_\S+?_available\(\))\s"					)
     for line in class_lines[start_idx : end_idx + 1]:
          _snake_case								=   r.search(_UpperCamelCase					)
          if backend_condition is not None:
               # replace the leading and trailing whitespaces to the space character " ".
               _snake_case								=   " " + backend_condition[0][1:-1] + " "
               _snake_case								=   r.sub(_UpperCamelCase      ,		_UpperCamelCase					)
               break

     if def_line is None:
          # `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
          # `all_model_classes` or `all_generative_model_classes`.
          _snake_case								=   end_idx
     else:
          # `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
          _snake_case								=   start_idx - 1
          # mark the lines of the currently existing `pipeline_model_mapping` to be removed.
          for idx in range(_UpperCamelCase      ,		end_idx + 1					):
               # These lines are going to be removed before writing to the test file.
               _snake_case								=   None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
     _snake_case								=   [x.__name__ for x in test_class.__bases__]
     if "PipelineTesterMixin" not in parent_classes:
          # Put `PipelineTesterMixin` just before `unittest.TestCase`
          _snake_case								=   [x for x in parent_classes if x != "TestCase"] + ["PipelineTesterMixin"]
          if "TestCase" in parent_classes:
               # Here we **assume** the original string is always with `unittest.TestCase`.
               _parent_classes.append("unittest.TestCase"					)
          _snake_case								=   ", ".join(_parent_classes					)
          for idx, line in enumerate(_UpperCamelCase					):
               # Find the ending of the declaration of `test_class`
               if line.strip().endswith("):"					):
                    # mark the lines of the declaration of `test_class` to be removed
                    for _idx in range(idx + 1					):
                         _snake_case								=   None  # noqa
                    break
        # Add the new, one-line, class declaration for `test_class`
          _snake_case								=   F"""class {test_class.__name__}({parent_classes}):\n"""

     # Add indentation
     _snake_case								=   " " * indent_level + line_to_add
     # Insert `pipeline_model_mapping` to `class_lines`.
     # (The line at `target_idx` should be kept by definition!)
     _snake_case								=   class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
     # Remove the lines that are marked to be removed
     _snake_case								=   [x for x in class_lines if x is not None]

     # Move from test class to module (in order to write to the test file)
     _snake_case								=   inspect.getsourcelines(inspect.getmodule(_UpperCamelCase					)					)[0]
     # Be careful with the 1-off between line numbers and array indices
     _snake_case								=   module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
     _snake_case								=   "".join(_UpperCamelCase					)

     _snake_case								=   inspect.getsourcefile(_UpperCamelCase					)
     with open(_UpperCamelCase      ,		"w"      ,		encoding="UTF-8"      ,		newline="\n"					) as fp:
          fp.write(_UpperCamelCase					)

     return line_to_add





def 			__a			(     _UpperCamelCase:					Optional[Any]      ,		_UpperCamelCase:					Dict=False					)		->				Tuple:

     """simple docstring"""




     _snake_case								=   find_test_class(_UpperCamelCase					)
     if test_class:
          add_pipeline_model_mapping(_UpperCamelCase      ,		overwrite=_UpperCamelCase					)


if __name__ == "__main__":
   UpperCamelCase_		:    Dict									=		argparse.ArgumentParser()
   parser.add_argument(
       '''--test_file''', type=str, help='''A path to the test file, starting with the repository\'s `tests` directory.'''
   )
   parser.add_argument(
       '''--all''',
       action='''store_true''',
       help='''If to check and modify all test files.''',
   )
   parser.add_argument(
       '''--overwrite''',
       action='''store_true''',
       help='''If to overwrite a test class if it has already defined `pipeline_model_mapping`.''',
   )
   UpperCamelCase_		:    Optional[Any]									=		parser.parse_args()

   if not args.all and not args.test_file:
      raise ValueError('''Please specify either `test_file` or pass `--all` to check/modify all test files.''')
   elif args.all and args.test_file:
      raise ValueError('''Only one of `--test_file` and `--all` could be specified.''')

   UpperCamelCase_		:    Dict									=		[]
   if args.test_file:
      UpperCamelCase_		:    Dict									=		[args.test_file]
   else:
      UpperCamelCase_		:    str									=		os.path.join('''tests''', '''models''', '''**''', '''test_modeling_*.py''')
      for test_file in glob.glob(pattern):
         # `Flax` is not concerned at this moment
         if not test_file.startswith('''test_modeling_flax_'''):
            test_files.append(test_file)

   for test_file in test_files:
      if test_file in TEST_FILE_TO_IGNORE:
         print(F'[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.')
         continue
      add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)








'''simple docstring'''





from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


UpperCamelCase_		:    Optional[int]									=		logging.get_logger(__name__)

UpperCamelCase_		:    str									=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_		:    List[str]									=		{
    '''vocab_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''openai/clip-vit-base-patch32''': (
            '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_		:    List[Any]									=		{
    '''openai/clip-vit-base-patch32''': 77,
}

class      _a       (		__lowerCAmelCase							):
     SCREAMING_SNAKE_CASE_							:					Any					       =		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_							:					List[Any]					       =		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_							:					Dict					       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_							:					int					       =		["""input_ids""", """attention_mask"""]
     SCREAMING_SNAKE_CASE_							:					Optional[Any]					       =		CLIPTokenizer






     def __init__(				self      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,_SCREAMING_SNAKE_CASE="<|startoftext|>"      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,**_SCREAMING_SNAKE_CASE      ,)							->    List[Any]:
          super().__init__(
              _SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE      ,tokenizer_file=_SCREAMING_SNAKE_CASE      ,unk_token=_SCREAMING_SNAKE_CASE      ,bos_token=_SCREAMING_SNAKE_CASE      ,eos_token=_SCREAMING_SNAKE_CASE      ,pad_token=_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE      ,)

          if not isinstance(self.backend_tokenizer.pre_tokenizer      ,pre_tokenizers.Sequence					):
               raise ValueError(
                   "The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"
                   " heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"
                   " to be compatible with this version.The easiest way to do so is"
                   " `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"
                   " to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"
                   " transformers."					)

          self._wrap_decode_method_backend_tokenizer()






     def        _lowercase			(				self					)							->    Tuple:
          _snake_case							=   self.backend_tokenizer.decode

          def new_decode_method(*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					):
               _snake_case							=   orig_decode_method(*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)
               _snake_case							=   text.replace(self.backend_tokenizer.model.end_of_word_suffix      ," "					).strip()
               return text

          _snake_case							=   new_decode_method






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case							=   [self.bos_token_id]
          _snake_case							=   [self.eos_token_id]

          if token_ids_a is None:
               return bos_token + token_ids_a + eos_token
          return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case							=   [self.bos_token_id]
          _snake_case							=   [self.eos_token_id]

          if token_ids_a is None:
               return len(bos_token + token_ids_a + eos_token					) * [0]
          return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token					) * [0]






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    Tuple[str]:
          _snake_case							=   self._tokenizer.model.save(_SCREAMING_SNAKE_CASE      ,name=_SCREAMING_SNAKE_CASE					)
          return tuple(_SCREAMING_SNAKE_CASE					)


