




'''simple docstring'''




import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
     import torch_xla.core.xla_model as xm





def   a_							(   __snake_case							:    str     , __snake_case							:    Any     )							->		int:



       """simple docstring"""
       if isinstance(__snake_case     , (list, tuple)     ):
              return honor_type(__snake_case     , (move_to_device(__snake_case     , __snake_case     ) for t in state)     )
       elif isinstance(__snake_case     , __snake_case     ):
              return type(__snake_case     )({k: move_to_device(__snake_case     , __snake_case     ) for k, v in state.items()}     )
       elif isinstance(__snake_case     , torch.Tensor     ):
              return state.to(__snake_case     )
       return state


class 						__UpperCamelCase (				torch.optim.Optimizer       ):






       def __init__(							self,	lowerCAmelCase,	lowerCAmelCase=True,	lowerCAmelCase=None						):



              """simple docstring"""







              lowerCamelCase_							=optimizer
              lowerCamelCase_							=scaler
              lowerCamelCase_							=AcceleratorState()
              lowerCamelCase_							=GradientState()
              lowerCamelCase_							=device_placement
              lowerCamelCase_							=False
              lowerCamelCase_							=None

              # Handle device placement
              if device_placement:
                     lowerCamelCase_							=self.optimizer.state_dict()
                     if self.accelerator_state.distributed_type == DistributedType.TPU:
                            xm.send_cpu_data_to_device(lowerCAmelCase,	self.accelerator_state.device						)
                     else:
                            lowerCamelCase_							=move_to_device(lowerCAmelCase,	self.accelerator_state.device						)
                     self.optimizer.load_state_dict(lowerCAmelCase						)






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              return self.optimizer.state






       @state.setter
       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=state






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              return self.optimizer.param_groups






       @param_groups.setter
       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=param_groups






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              return self.optimizer.defaults






       @defaults.setter
       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=defaults






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              self.optimizer.add_param_group(lowerCAmelCase						)






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
                     xm.send_cpu_data_to_device(lowerCAmelCase,	self.accelerator_state.device						)
              self.optimizer.load_state_dict(lowerCAmelCase						)






       def   lowercase__  (							self						):



              """simple docstring"""







              return self.optimizer.state_dict()






       def   lowercase__  (							self,	lowerCAmelCase=None						):



              """simple docstring"""







              if self.gradient_state.sync_gradients:
                     lowerCamelCase_							='''set_to_none''' in inspect.signature(self.optimizer.zero_grad						).parameters
                     if accept_arg:
                            if set_to_none is None:
                                   lowerCamelCase_							=False
                            self.optimizer.zero_grad(set_to_none=lowerCAmelCase						)
                     else:
                            if set_to_none is not None:
                                   raise ValueError('''`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.'''						)
                            self.optimizer.zero_grad()






       def   lowercase__  (							self,	lowerCAmelCase=None						):



              """simple docstring"""







              if self.gradient_state.sync_gradients:
                     if self.accelerator_state.distributed_type == DistributedType.TPU:
                            lowerCamelCase_							={'''closure''': closure} if closure is not None else {}
                            xm.optimizer_step(self.optimizer,	optimizer_args=lowerCAmelCase						)
                     elif self.scaler is not None:
                            lowerCamelCase_							=False
                            if self._last_scale is None:
                                   # `get_scale` is an async operation requiring full synchronization
                                   # on CPU and GPUs before finishing. As a result, we store away
                                   # the prior one to reduce the call overhead
                                   lowerCamelCase_							=self.scaler.get_scale()
                                   lowerCamelCase_							=True
                            self.scaler.step(self.optimizer,	lowerCAmelCase						)
                            self.scaler.update()
                            lowerCamelCase_							=self.scaler.get_scale()
                            if not new_scale:
                                   # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
                                   lowerCamelCase_							=scale_after < self._last_scale
                            lowerCamelCase_							=scale_after

                     else:
                            self.optimizer.step(lowerCAmelCase						)






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              for param_group in self.optimizer.param_groups:
                     lowerCamelCase_							=[parameters_map.get(lowerCAmelCase,	lowerCAmelCase						) for p in param_group['''params''']]






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              warnings.warn(
                  '''The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '''
                  '''`optimizer.step_was_skipped` instead.''',	lowerCAmelCase,	)
              return self._is_overflow






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              return self._is_overflow






       def __getstate__(							self						):



              """simple docstring"""







              return self.__dict__.copy()






       def __setstate__(							self,	lowerCAmelCase						):



              """simple docstring"""







              self.__dict__.update(lowerCAmelCase						)










'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   :	Union[str, Any]							   = {
    """configuration_openai""": ["""OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OpenAIGPTConfig"""],
    """tokenization_openai""": ["""OpenAIGPTTokenizer"""],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Optional[int]							   = ["""OpenAIGPTTokenizerFast"""]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	int							   = [
         """OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """OpenAIGPTDoubleHeadsModel""",
         """OpenAIGPTForSequenceClassification""",
         """OpenAIGPTLMHeadModel""",
         """OpenAIGPTModel""",
         """OpenAIGPTPreTrainedModel""",
         """load_tf_weights_in_openai_gpt""",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Dict							   = [
         """TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """TFOpenAIGPTDoubleHeadsModel""",
         """TFOpenAIGPTForSequenceClassification""",
         """TFOpenAIGPTLMHeadModel""",
         """TFOpenAIGPTMainLayer""",
         """TFOpenAIGPTModel""",
         """TFOpenAIGPTPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
     from .tokenization_openai import OpenAIGPTTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_openai_fast import OpenAIGPTTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_openai import (
              OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              OpenAIGPTDoubleHeadsModel,
              OpenAIGPTForSequenceClassification,
              OpenAIGPTLMHeadModel,
              OpenAIGPTModel,
              OpenAIGPTPreTrainedModel,
              load_tf_weights_in_openai_gpt,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_openai import (
              TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFOpenAIGPTDoubleHeadsModel,
              TFOpenAIGPTForSequenceClassification,
              TFOpenAIGPTLMHeadModel,
              TFOpenAIGPTMainLayer,
              TFOpenAIGPTModel,
              TFOpenAIGPTPreTrainedModel,
          )

else:
     import sys

     a_   :	List[Any]							   = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




