






import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


A_	:	Optional[int]				=				logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
A_	:	Optional[Any]				=				direct_transformers_import(Path(__file__).parent)


A_	:	Optional[Any]				=				{
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       a       :	Optional[Any]								=['''feature_extractor''', '''tokenizer''']
       # Names need to be attr_class for attr in attributes
       a       :	Any								=None
       a       :	List[Any]								=None
       a       :	Tuple								=None





       def __init__(      self						,    *_lowerCamelCase						,    **_lowerCamelCase							):
         # Sanitize args and kwargs
         for key in kwargs:
           if key not in self.attributes:
             raise TypeError(f'''Unexpected keyword argument {key}.'''							)
         for arg, attribute_name in zip(_lowerCamelCase						,    self.attributes							):
           if attribute_name in kwargs:
             raise TypeError(f'''Got multiple values for argument {attribute_name}.'''							)
           else:
             UpperCamelCase_:      int					   =       arg

         if len(_lowerCamelCase							) != len(self.attributes							):
           raise ValueError(
               f'''This processor requires {len(self.attributes							)} arguments: {', '.join(self.attributes							)}. Got '''
               f'''{len(_lowerCamelCase							)} arguments instead.'''							)

         # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
         for attribute_name, arg in kwargs.items():
           UpperCamelCase_:      Tuple					   =       getattr(self						,    f'''{attribute_name}_class'''							)
           # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
           UpperCamelCase_:      str					   =       AUTO_TO_BASE_CLASS_MAPPING.get(_lowerCamelCase						,    _lowerCamelCase							)
           if isinstance(_lowerCamelCase						,    _lowerCamelCase							):
             UpperCamelCase_:      Tuple					   =       tuple(getattr(_lowerCamelCase						,    _lowerCamelCase							) for n in class_name if n is not None							)
           else:
             UpperCamelCase_:      int					   =       getattr(_lowerCamelCase						,    _lowerCamelCase							)

           if not isinstance(_lowerCamelCase						,    _lowerCamelCase							):
             raise ValueError(
                 f'''Received a {type(_lowerCamelCase							).__name__} for argument {attribute_name}, but a {class_name} was expected.'''							)

           setattr(self						,    _lowerCamelCase						,    _lowerCamelCase							)





       def __repr__(      self							):
         UpperCamelCase_:      Optional[int]					   =       [f'''- {name}: {repr(getattr(self						,    _lowerCamelCase							)							)}''' for name in self.attributes]
         UpperCamelCase_:      str					   =       '\n'.join(_lowerCamelCase							)
         return f'''{self.__class__.__name__}:\n{attributes_repr}'''





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase = False						,    **_lowerCamelCase							):
         os.makedirs(_lowerCamelCase						,    exist_ok=_lowerCamelCase							)

         if push_to_hub:
           UpperCamelCase_:      Dict					   =       kwargs.pop('commit_message'						,    _lowerCamelCase							)
           UpperCamelCase_:      Dict					   =       kwargs.pop('repo_id'						,    save_directory.split(os.path.sep							)[-1]							)
           UpperCamelCase_:      Any					   =       self._create_repo(_lowerCamelCase						,    **_lowerCamelCase							)
           UpperCamelCase_:      Optional[int]					   =       self._get_files_timestamps(_lowerCamelCase							)
         # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
         # loaded from the Hub.
         if self._auto_class is not None:
           UpperCamelCase_:      Tuple					   =       [getattr(self						,    _lowerCamelCase							) for attribute_name in self.attributes]
           UpperCamelCase_:      List[str]					   =       [(a.init_kwargs if isinstance(_lowerCamelCase						,    _lowerCamelCase							) else a) for a in attrs]
           custom_object_save(self						,    _lowerCamelCase						,    config=_lowerCamelCase							)

         for attribute_name in self.attributes:
           UpperCamelCase_:      str					   =       getattr(self						,    _lowerCamelCase							)
           # Include the processor class in the attribute config so this processor can then be reloaded with the
           # `AutoProcessor` API.
           if hasattr(_lowerCamelCase						,    '_set_processor_class'							):
             attribute._set_processor_class(self.__class__.__name__							)
           attribute.save_pretrained(_lowerCamelCase							)

         if self._auto_class is not None:
           # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
           for attribute_name in self.attributes:
             UpperCamelCase_:      Optional[Any]					   =       getattr(self						,    _lowerCamelCase							)
             if isinstance(_lowerCamelCase						,    _lowerCamelCase							):
               del attribute.init_kwargs["auto_map"]

         if push_to_hub:
           self._upload_modified_files(
               _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    commit_message=_lowerCamelCase						,    token=kwargs.get('use_auth_token'							)						,    )





       @classmethod
       def     _a					(      cls						,    _lowerCamelCase						,    _lowerCamelCase = None						,    _lowerCamelCase = False						,    _lowerCamelCase = False						,    _lowerCamelCase = None						,    _lowerCamelCase = "main"						,    **_lowerCamelCase						,    ):
         UpperCamelCase_:      str					   =       cache_dir
         UpperCamelCase_:      Tuple					   =       force_download
         UpperCamelCase_:      Optional[int]					   =       local_files_only
         UpperCamelCase_:      List[Any]					   =       revision

         UpperCamelCase_:      Any					   =       kwargs.pop('use_auth_token'						,    _lowerCamelCase							)
         if use_auth_token is not None:
           warnings.warn(
               'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'						,    _lowerCamelCase							)
           if token is not None:
             raise ValueError(
                 '`token` and `use_auth_token` are both specified. Please set only the argument `token`.'							)
           UpperCamelCase_:      Union[str, Any]					   =       use_auth_token

         if token is not None:
           # change to `token` in a follow-up PR
           UpperCamelCase_:      List[str]					   =       token

         UpperCamelCase_:      Optional[int]					   =       cls._get_arguments_from_pretrained(_lowerCamelCase						,    **_lowerCamelCase							)
         return cls(*_lowerCamelCase							)





       @classmethod
       def     _a					(      cls						,    _lowerCamelCase="AutoProcessor"							):
         if not isinstance(_lowerCamelCase						,    _lowerCamelCase							):
           UpperCamelCase_:      List[Any]					   =       auto_class.__name__

         import transformers.models.auto as auto_module

         if not hasattr(_lowerCamelCase						,    _lowerCamelCase							):
           raise ValueError(f'''{auto_class} is not a valid auto class.'''							)

         UpperCamelCase_:      Optional[Any]					   =       auto_class





       @classmethod
       def     _a					(      cls						,    _lowerCamelCase						,    **_lowerCamelCase							):
         UpperCamelCase_:      Union[str, Any]					   =       []
         for attribute_name in cls.attributes:
           UpperCamelCase_:      Tuple					   =       getattr(cls						,    f'''{attribute_name}_class'''							)
           if isinstance(_lowerCamelCase						,    _lowerCamelCase							):
             UpperCamelCase_:      List[Any]					   =       tuple(getattr(_lowerCamelCase						,    _lowerCamelCase							) if n is not None else None for n in class_name							)
             UpperCamelCase_:      List[Any]					   =       kwargs.get('use_fast'						,    _lowerCamelCase							)
             if use_fast and classes[1] is not None:
               UpperCamelCase_:      int					   =       classes[1]
             else:
               UpperCamelCase_:      Union[str, Any]					   =       classes[0]
           else:
             UpperCamelCase_:      Tuple					   =       getattr(_lowerCamelCase						,    _lowerCamelCase							)

           args.append(attribute_class.from_pretrained(_lowerCamelCase						,    **_lowerCamelCase							)							)
         return args





       @property
       def     _a					(      self							):
         UpperCamelCase_:      int					   =       getattr(self						,    self.attributes[0]							)
         return getattr(_lowerCamelCase						,    'model_input_names'						,    _lowerCamelCase							)







A_	:	Optional[Any]				=				copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
   A_	:	Tuple				=				ProcessorMixin.push_to_hub.__doc__.format(
       object='processor', object_class='AutoProcessor', object_files='processor files'
   )







# Lint as: python3

import hashlib
from typing import Union

def 						snake_case      (UpperCAmelCase__						) ->       bytes:
  if isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    # Data already in bytes, returns as it as
    return hash_data
  elif isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    # We keep the data as it as for it ot be later encoded to UTF-8
    # However replace `\\` with `/` for Windows compatibility
    UpperCamelCase_:      List[str]					   =       hash_data.replace('\\'			,					'/'						)
  elif isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    UpperCamelCase_:      Union[str, Any]					   =       str(UpperCAmelCase__						)
  else:
    # If data is not of the required type, raise error
    raise InvalidKeyError(UpperCAmelCase__						)

  return hash_data.encode('utf-8'						)




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      int					   =       '\nFAILURE TO GENERATE DATASET: Invalid key type detected'
         UpperCamelCase_:      Dict					   =       f'''\nFound Key {hash_data} of type {type(_lowerCamelCase							)}'''
         UpperCamelCase_:      Any					   =       '\nKeys should be either str, int or bytes type'
         super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)







class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase=""							):
         UpperCamelCase_:      int					   =       key
         UpperCamelCase_:      Dict					   =       duplicate_key_indices
         UpperCamelCase_:      int					   =       fix_msg
         UpperCamelCase_:      List[Any]					   =       'Found multiple examples generated with the same key'
         if len(_lowerCamelCase							) <= 2_0:
           UpperCamelCase_:      Tuple					   =       f'''\nThe examples at index {', '.join(_lowerCamelCase							)} have the key {key}'''
         else:
           UpperCamelCase_:      Optional[int]					   =       f'''\nThe examples at index {', '.join(duplicate_key_indices[:2_0]							)}... ({len(_lowerCamelCase							) - 2_0} more) have the key {key}'''
         UpperCamelCase_:      List[Any]					   =       '\n' + fix_msg if fix_msg else ''
         super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)







class        _lowerCAmelCase:





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      int					   =       hashlib.mda(_as_bytes(_lowerCamelCase							)							)





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      Optional[int]					   =       self._split_mda.copy()
         UpperCamelCase_:      str					   =       _as_bytes(_lowerCamelCase							)
         mda.update(_lowerCamelCase							)
         # Convert to integer with hexadecimal conversion
         return int(mda.hexdigest()						,    1_6							)