

'''simple docstring'''



import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCAmelCase				: Optional[Any]         =  logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCAmelCase				: Tuple         =  direct_transformers_import(Path(__file__).parent)


UpperCAmelCase				: Tuple         =  {
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}







class 		lowerCAmelCase__     (   a      ):



 """simple docstring"""



 lowerCAmelCase__						      =   ["feature_extractor", "tokenizer"]
 # Names need to be attr_class for attr in attributes
 lowerCAmelCase__						      =   None
 lowerCAmelCase__						      =   None
 lowerCAmelCase__						      =   None
 def __init__(	self      : int      ,							*__SCREAMING_SNAKE_CASE      : Tuple      ,							**__SCREAMING_SNAKE_CASE      : Optional[int]    )						->				int:


      """simple docstring"""




      for key in kwargs:
           if key not in self.attributes:
                raise TypeError(f'Unexpected keyword argument {key}.'    )
      for arg, attribute_name in zip(__SCREAMING_SNAKE_CASE      ,							self.attributes    ):
           if attribute_name in kwargs:
                raise TypeError(f'Got multiple values for argument {attribute_name}.'    )
           else:
                __SCREAMING_SNAKE_CASE						     =   arg

      if len(__SCREAMING_SNAKE_CASE    ) != len(self.attributes    ):
           raise ValueError(
               f'This processor requires {len(self.attributes    )} arguments: {", ".join(self.attributes    )}. Got '
               f'{len(__SCREAMING_SNAKE_CASE    )} arguments instead.'    )

      # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
      for attribute_name, arg in kwargs.items():
           __SCREAMING_SNAKE_CASE						     =   getattr(self      ,							f'{attribute_name}_class'    )
           # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
           __SCREAMING_SNAKE_CASE						     =   AUTO_TO_BASE_CLASS_MAPPING.get(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    )
           if isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
                __SCREAMING_SNAKE_CASE						     =   tuple(getattr(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ) for n in class_name if n is not None    )
           else:
                __SCREAMING_SNAKE_CASE						     =   getattr(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    )

           if not isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
                raise ValueError(
                    f'Received a {type(__SCREAMING_SNAKE_CASE    ).__name__} for argument {attribute_name}, but a {class_name} was expected.'    )

           setattr(self      ,							__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    )
 def __repr__(	self      : Optional[Any]    )						->				List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   [f'- {name}: {repr(getattr(self      ,							__SCREAMING_SNAKE_CASE    )    )}' for name in self.attributes]
      __SCREAMING_SNAKE_CASE						     =   """\n""".join(__SCREAMING_SNAKE_CASE    )
      return f'{self.__class__.__name__}:\n{attributes_repr}'
 def    UpperCAmelCase__	(	self      : Optional[Any]      ,							__SCREAMING_SNAKE_CASE      : Optional[int]      ,							__SCREAMING_SNAKE_CASE      : bool = False      ,							**__SCREAMING_SNAKE_CASE      : Tuple    )						->				str:


      """simple docstring"""




      os.makedirs(__SCREAMING_SNAKE_CASE      ,							exist_ok=__SCREAMING_SNAKE_CASE    )

      if push_to_hub:
           __SCREAMING_SNAKE_CASE						     =   kwargs.pop("""commit_message"""      ,							__SCREAMING_SNAKE_CASE    )
           __SCREAMING_SNAKE_CASE						     =   kwargs.pop("""repo_id"""      ,							save_directory.split(os.path.sep    )[-1]    )
           __SCREAMING_SNAKE_CASE						     =   self._create_repo(__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE    )
           __SCREAMING_SNAKE_CASE						     =   self._get_files_timestamps(__SCREAMING_SNAKE_CASE    )
      # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
      # loaded from the Hub.
      if self._auto_class is not None:
           __SCREAMING_SNAKE_CASE						     =   [getattr(self      ,							__SCREAMING_SNAKE_CASE    ) for attribute_name in self.attributes]
           __SCREAMING_SNAKE_CASE						     =   [(a.init_kwargs if isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ) else a) for a in attrs]
           custom_object_save(self      ,							__SCREAMING_SNAKE_CASE      ,							config=__SCREAMING_SNAKE_CASE    )

      for attribute_name in self.attributes:
           __SCREAMING_SNAKE_CASE						     =   getattr(self      ,							__SCREAMING_SNAKE_CASE    )
           # Include the processor class in the attribute config so this processor can then be reloaded with the
           # `AutoProcessor` API.
           if hasattr(__SCREAMING_SNAKE_CASE      ,							"""_set_processor_class"""    ):
                attribute._set_processor_class(self.__class__.__name__    )
           attribute.save_pretrained(__SCREAMING_SNAKE_CASE    )

      if self._auto_class is not None:
           # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
           for attribute_name in self.attributes:
                __SCREAMING_SNAKE_CASE						     =   getattr(self      ,							__SCREAMING_SNAKE_CASE    )
                if isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
                     del attribute.init_kwargs["auto_map"]

      if push_to_hub:
           self._upload_modified_files(
               __SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE      ,							commit_message=__SCREAMING_SNAKE_CASE      ,							token=kwargs.get("""use_auth_token"""    )      ,							)
 @classmethod
 def    UpperCAmelCase__	(	cls      : Tuple      ,							__SCREAMING_SNAKE_CASE      : Union[str, os.PathLike]      ,							__SCREAMING_SNAKE_CASE      : Optional[Union[str, os.PathLike]] = None      ,							__SCREAMING_SNAKE_CASE      : bool = False      ,							__SCREAMING_SNAKE_CASE      : bool = False      ,							__SCREAMING_SNAKE_CASE      : Optional[Union[str, bool]] = None      ,							__SCREAMING_SNAKE_CASE      : str = "main"      ,							**__SCREAMING_SNAKE_CASE      : Tuple      ,							)						->				Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   cache_dir
      __SCREAMING_SNAKE_CASE						     =   force_download
      __SCREAMING_SNAKE_CASE						     =   local_files_only
      __SCREAMING_SNAKE_CASE						     =   revision

      __SCREAMING_SNAKE_CASE						     =   kwargs.pop("""use_auth_token"""      ,							__SCREAMING_SNAKE_CASE    )
      if use_auth_token is not None:
           warnings.warn(
               """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""      ,							__SCREAMING_SNAKE_CASE    )
           if token is not None:
                raise ValueError(
                    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""    )
           __SCREAMING_SNAKE_CASE						     =   use_auth_token

      if token is not None:
           # change to `token` in a follow-up PR
           __SCREAMING_SNAKE_CASE						     =   token

      __SCREAMING_SNAKE_CASE						     =   cls._get_arguments_from_pretrained(__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE    )
      return cls(*__SCREAMING_SNAKE_CASE    )
 @classmethod
 def    UpperCAmelCase__	(	cls      : Dict      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]="AutoProcessor"    )						->				Any:


      """simple docstring"""




      if not isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
           __SCREAMING_SNAKE_CASE						     =   auto_class.__name__

      import transformers.models.auto as auto_module

      if not hasattr(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
           raise ValueError(f'{auto_class} is not a valid auto class.'    )

      __SCREAMING_SNAKE_CASE						     =   auto_class
 @classmethod
 def    UpperCAmelCase__	(	cls      : Any      ,							__SCREAMING_SNAKE_CASE      : Any      ,							**__SCREAMING_SNAKE_CASE      : List[Any]    )						->				List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   []
      for attribute_name in cls.attributes:
           __SCREAMING_SNAKE_CASE						     =   getattr(cls      ,							f'{attribute_name}_class'    )
           if isinstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ):
                __SCREAMING_SNAKE_CASE						     =   tuple(getattr(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ) if n is not None else None for n in class_name    )
                __SCREAMING_SNAKE_CASE						     =   kwargs.get("""use_fast"""      ,							__SCREAMING_SNAKE_CASE    )
                if use_fast and classes[1] is not None:
                     __SCREAMING_SNAKE_CASE						     =   classes[1]
                else:
                     __SCREAMING_SNAKE_CASE						     =   classes[0]
           else:
                __SCREAMING_SNAKE_CASE						     =   getattr(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    )

           args.append(attribute_class.from_pretrained(__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE    )    )
      return args


 @property
 def    UpperCAmelCase__	(	self      : List[str]    )						->				Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   getattr(self      ,							self.attributes[0]    )
      return getattr(__SCREAMING_SNAKE_CASE      ,							"""model_input_names"""      ,							__SCREAMING_SNAKE_CASE    )


UpperCAmelCase				: Union[str, Any]         =  copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
  UpperCAmelCase				: List[Any]         =  ProcessorMixin.push_to_hub.__doc__.format(
      object='processor', object_class='AutoProcessor', object_files='processor files'
  )









'''simple docstring'''



import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm







def      a__ (						a__   ,					a__		):




     """simple docstring"""


     if isinstance(a__   ,					(list, tuple)		):
          return honor_type(a__   ,					(move_to_device(a__   ,					a__		) for t in state)		)
     elif isinstance(a__   ,					a__		):
          return type(a__		)({k: move_to_device(a__   ,					a__		) for k, v in state.items()}		)
     elif isinstance(a__   ,					torch.Tensor		):
          return state.to(a__		)
     return state







class 		lowerCAmelCase__     (   torch.optim.Optimizer      ):



 """simple docstring"""



 def __init__(	self      : Tuple      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]      ,							__SCREAMING_SNAKE_CASE      : Dict=True      ,							__SCREAMING_SNAKE_CASE      : List[Any]=None    )						->				str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   optimizer
      __SCREAMING_SNAKE_CASE						     =   scaler
      __SCREAMING_SNAKE_CASE						     =   AcceleratorState()
      __SCREAMING_SNAKE_CASE						     =   GradientState()
      __SCREAMING_SNAKE_CASE						     =   device_placement
      __SCREAMING_SNAKE_CASE						     =   False
      __SCREAMING_SNAKE_CASE						     =   None

      # Handle device placement
      if device_placement:
           __SCREAMING_SNAKE_CASE						     =   self.optimizer.state_dict()
           if self.accelerator_state.distributed_type == DistributedType.TPU:
                xm.send_cpu_data_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
           else:
                __SCREAMING_SNAKE_CASE						     =   move_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
           self.optimizer.load_state_dict(__SCREAMING_SNAKE_CASE    )
 @property
 def    UpperCAmelCase__	(	self      : str    )						->				Union[str, Any]:


      """simple docstring"""




      return self.optimizer.state
 @state.setter
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]    )						->				List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   state
 @property
 def    UpperCAmelCase__	(	self      : Optional[int]    )						->				str:


      """simple docstring"""




      return self.optimizer.param_groups
 @param_groups.setter
 def    UpperCAmelCase__	(	self      : Dict      ,							__SCREAMING_SNAKE_CASE      : str    )						->				List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   param_groups
 @property
 def    UpperCAmelCase__	(	self      : List[Any]    )						->				Any:


      """simple docstring"""




      return self.optimizer.defaults
 @defaults.setter
 def    UpperCAmelCase__	(	self      : Any      ,							__SCREAMING_SNAKE_CASE      : Dict    )						->				Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   defaults
 def    UpperCAmelCase__	(	self      : Any      ,							__SCREAMING_SNAKE_CASE      : Any    )						->				List[str]:


      """simple docstring"""




      self.optimizer.add_param_group(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[str]    )						->				Dict:


      """simple docstring"""




      if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
           xm.send_cpu_data_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
      self.optimizer.load_state_dict(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : Union[str, Any]    )						->				Union[str, Any]:


      """simple docstring"""




      return self.optimizer.state_dict()
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[str]=None    )						->				Any:


      """simple docstring"""




      if self.gradient_state.sync_gradients:
           __SCREAMING_SNAKE_CASE						     =   """set_to_none""" in inspect.signature(self.optimizer.zero_grad    ).parameters
           if accept_arg:
                if set_to_none is None:
                     __SCREAMING_SNAKE_CASE						     =   False
                self.optimizer.zero_grad(set_to_none=__SCREAMING_SNAKE_CASE    )
           else:
                if set_to_none is not None:
                     raise ValueError("""`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer."""    )
                self.optimizer.zero_grad()
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[Any]=None    )						->				Dict:


      """simple docstring"""




      if self.gradient_state.sync_gradients:
           if self.accelerator_state.distributed_type == DistributedType.TPU:
                __SCREAMING_SNAKE_CASE						     =   {"""closure""": closure} if closure is not None else {}
                xm.optimizer_step(self.optimizer      ,							optimizer_args=__SCREAMING_SNAKE_CASE    )
           elif self.scaler is not None:
                __SCREAMING_SNAKE_CASE						     =   False
                if self._last_scale is None:
                     # `get_scale` is an async operation requiring full synchronization
                     # on CPU and GPUs before finishing. As a result, we store away
                     # the prior one to reduce the call overhead
                     __SCREAMING_SNAKE_CASE						     =   self.scaler.get_scale()
                     __SCREAMING_SNAKE_CASE						     =   True
                self.scaler.step(self.optimizer      ,							__SCREAMING_SNAKE_CASE    )
                self.scaler.update()
                __SCREAMING_SNAKE_CASE						     =   self.scaler.get_scale()
                if not new_scale:
                     # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
                     __SCREAMING_SNAKE_CASE						     =   scale_after < self._last_scale
                __SCREAMING_SNAKE_CASE						     =   scale_after

           else:
                self.optimizer.step(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : Union[str, Any]      ,							__SCREAMING_SNAKE_CASE      : Any    )						->				Any:


      """simple docstring"""




      for param_group in self.optimizer.param_groups:
           __SCREAMING_SNAKE_CASE						     =   [parameters_map.get(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ) for p in param_group["""params"""]]
 @property
 def    UpperCAmelCase__	(	self      : int    )						->				Any:


      """simple docstring"""




      warnings.warn(
          """The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use """
          """`optimizer.step_was_skipped` instead."""      ,							__SCREAMING_SNAKE_CASE      ,							)
      return self._is_overflow
 @property
 def    UpperCAmelCase__	(	self      : Union[str, Any]    )						->				Optional[int]:


      """simple docstring"""




      return self._is_overflow
 def __getstate__(	self      : Tuple    )						->				str:


      """simple docstring"""




      return self.__dict__.copy()


 def __setstate__(	self      : str      ,							__SCREAMING_SNAKE_CASE      : Optional[int]    )						->				Any:


      """simple docstring"""




      self.__dict__.update(__SCREAMING_SNAKE_CASE    )






