

'''simple docstring'''



import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
      SpeechaTextaDecoder,
      SpeechaTextaForCausalLM,
  )







@require_torch
class 		lowerCAmelCase__     :



 """simple docstring"""



 def __init__(	self      : Dict      ,							__SCREAMING_SNAKE_CASE      : str      ,							__SCREAMING_SNAKE_CASE      : str=99      ,							__SCREAMING_SNAKE_CASE      : List[Any]=13      ,							__SCREAMING_SNAKE_CASE      : str=16      ,							__SCREAMING_SNAKE_CASE      : Any=7      ,							__SCREAMING_SNAKE_CASE      : Dict=True      ,							__SCREAMING_SNAKE_CASE      : str=True      ,							__SCREAMING_SNAKE_CASE      : int=True      ,							__SCREAMING_SNAKE_CASE      : List[str]=False      ,							__SCREAMING_SNAKE_CASE      : Optional[int]=True      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]=2      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]=32      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]=4      ,							__SCREAMING_SNAKE_CASE      : List[Any]=4      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]=30      ,							__SCREAMING_SNAKE_CASE      : str=0      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]=1      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]=2      ,							__SCREAMING_SNAKE_CASE      : List[Any]=None      ,							)						->				List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   parent
      __SCREAMING_SNAKE_CASE						     =   batch_size
      __SCREAMING_SNAKE_CASE						     =   decoder_seq_length
      # For common tests
      __SCREAMING_SNAKE_CASE						     =   self.decoder_seq_length
      __SCREAMING_SNAKE_CASE						     =   is_training
      __SCREAMING_SNAKE_CASE						     =   use_attention_mask
      __SCREAMING_SNAKE_CASE						     =   use_labels

      __SCREAMING_SNAKE_CASE						     =   vocab_size
      __SCREAMING_SNAKE_CASE						     =   d_model
      __SCREAMING_SNAKE_CASE						     =   d_model
      __SCREAMING_SNAKE_CASE						     =   decoder_layers
      __SCREAMING_SNAKE_CASE						     =   decoder_layers
      __SCREAMING_SNAKE_CASE						     =   decoder_ffn_dim
      __SCREAMING_SNAKE_CASE						     =   decoder_attention_heads
      __SCREAMING_SNAKE_CASE						     =   decoder_attention_heads
      __SCREAMING_SNAKE_CASE						     =   eos_token_id
      __SCREAMING_SNAKE_CASE						     =   bos_token_id
      __SCREAMING_SNAKE_CASE						     =   pad_token_id
      __SCREAMING_SNAKE_CASE						     =   decoder_start_token_id
      __SCREAMING_SNAKE_CASE						     =   use_cache
      __SCREAMING_SNAKE_CASE						     =   max_position_embeddings

      __SCREAMING_SNAKE_CASE						     =   None
      __SCREAMING_SNAKE_CASE						     =   decoder_seq_length
      __SCREAMING_SNAKE_CASE						     =   2
      __SCREAMING_SNAKE_CASE						     =   1
 def    UpperCAmelCase__	(	self      : Tuple    )						->				Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   ids_tensor([self.batch_size, self.decoder_seq_length]      ,							self.vocab_size    )

      __SCREAMING_SNAKE_CASE						     =   None
      if self.use_attention_mask:
           __SCREAMING_SNAKE_CASE						     =   ids_tensor([self.batch_size, self.decoder_seq_length]      ,							vocab_size=2    )

      __SCREAMING_SNAKE_CASE						     =   None
      if self.use_labels:
           __SCREAMING_SNAKE_CASE						     =   ids_tensor([self.batch_size, self.decoder_seq_length]      ,							self.vocab_size    )

      __SCREAMING_SNAKE_CASE						     =   SpeechaTextaConfig(
          vocab_size=self.vocab_size      ,							d_model=self.d_model      ,							decoder_layers=self.decoder_layers      ,							decoder_ffn_dim=self.decoder_ffn_dim      ,							decoder_attention_heads=self.decoder_attention_heads      ,							eos_token_id=self.eos_token_id      ,							bos_token_id=self.bos_token_id      ,							use_cache=self.use_cache      ,							pad_token_id=self.pad_token_id      ,							decoder_start_token_id=self.decoder_start_token_id      ,							max_position_embeddings=self.max_position_embeddings      ,							)

      return (
          config,
          input_ids,
          attention_mask,
          lm_labels,
      )
 def    UpperCAmelCase__	(	self      : List[Any]      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]      ,							__SCREAMING_SNAKE_CASE      : int      ,							__SCREAMING_SNAKE_CASE      : List[str]      ,							__SCREAMING_SNAKE_CASE      : Any      ,							)						->				Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   True
      __SCREAMING_SNAKE_CASE						     =   SpeechaTextaDecoder(config=__SCREAMING_SNAKE_CASE    ).to(__SCREAMING_SNAKE_CASE    ).eval()
      __SCREAMING_SNAKE_CASE						     =   input_ids[:2]

      input_ids[input_ids == 0] += 1
      # first forward pass
      __SCREAMING_SNAKE_CASE						     =   model(__SCREAMING_SNAKE_CASE      ,							use_cache=__SCREAMING_SNAKE_CASE    )
      __SCREAMING_SNAKE_CASE						     =   model(__SCREAMING_SNAKE_CASE    )
      __SCREAMING_SNAKE_CASE						     =   model(__SCREAMING_SNAKE_CASE      ,							use_cache=__SCREAMING_SNAKE_CASE    )

      self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE    ) == len(__SCREAMING_SNAKE_CASE    )    )
      self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE    ) == len(__SCREAMING_SNAKE_CASE    ) + 1    )

      __SCREAMING_SNAKE_CASE						     =   outputs["""past_key_values"""]

      # create hypothetical next token and extent to next_input_ids
      __SCREAMING_SNAKE_CASE						     =   ids_tensor((2, 1)      ,							config.vocab_size - 1    ) + 1

      # append to next input_ids and
      __SCREAMING_SNAKE_CASE						     =   torch.cat([input_ids, next_tokens]      ,							dim=-1    )

      print(__SCREAMING_SNAKE_CASE    )

      __SCREAMING_SNAKE_CASE						     =   model(__SCREAMING_SNAKE_CASE    )["""last_hidden_state"""]
      __SCREAMING_SNAKE_CASE						     =   model(__SCREAMING_SNAKE_CASE      ,							past_key_values=__SCREAMING_SNAKE_CASE    )["""last_hidden_state"""]

      # select random slice
      __SCREAMING_SNAKE_CASE						     =   ids_tensor((1,)      ,							output_from_past.shape[-1]    ).item()
      __SCREAMING_SNAKE_CASE						     =   output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
      __SCREAMING_SNAKE_CASE						     =   output_from_past[:, 0, random_slice_idx].detach()

      # test that outputs are equal for slice
      assert torch.allclose(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE      ,							atol=1E-3    )


 def    UpperCAmelCase__	(	self      : Optional[Any]    )						->				Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   self.prepare_config_and_inputs()
      (
          (
          __SCREAMING_SNAKE_CASE    
      )     ,    (
          __SCREAMING_SNAKE_CASE    
      )     ,    (
          __SCREAMING_SNAKE_CASE    
      )     ,    (
          __SCREAMING_SNAKE_CASE    
      )     ,        
      )						     =   config_and_inputs

      __SCREAMING_SNAKE_CASE						     =   {
          """input_ids""": input_ids,
          """attention_mask""": attention_mask,
      }
      return config, inputs_dict







@require_torch
class 		lowerCAmelCase__     (   a				,      a				,      a				,      unittest.TestCase      ):



 """simple docstring"""



 lowerCAmelCase__						      =   (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
 lowerCAmelCase__						      =   (SpeechaTextaForCausalLM,) if is_torch_available() else ()
 lowerCAmelCase__						      =   {"text-generation": SpeechaTextaForCausalLM} if is_torch_available() else {}
 lowerCAmelCase__						      =   True
 lowerCAmelCase__						      =   False
 def    UpperCAmelCase__	(	self      : List[Any]      ,							)						->				Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   SpeechaTextaStandaloneDecoderModelTester(self      ,							is_training=__SCREAMING_SNAKE_CASE    )
      __SCREAMING_SNAKE_CASE						     =   ConfigTester(self      ,							config_class=__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : int    )						->				Optional[Any]:


      """simple docstring"""




      pass
 def    UpperCAmelCase__	(	self      : Tuple    )						->				int:


      """simple docstring"""




      pass
 def    UpperCAmelCase__	(	self      : List[str]    )						->				List[str]:


      """simple docstring"""




      pass
 def    UpperCAmelCase__	(	self      : int    )						->				Dict:


      """simple docstring"""




      self.config_tester.run_common_tests()
 def    UpperCAmelCase__	(	self      : Tuple    )						->				Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_decoder_model_past(*__SCREAMING_SNAKE_CASE    )


 def    UpperCAmelCase__	(	self      : Optional[int]    )						->				int:


      """simple docstring"""




      return








'''simple docstring'''



import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm







def      a__ (						a__   ,					a__		):




     """simple docstring"""


     if isinstance(a__   ,					(list, tuple)		):
          return honor_type(a__   ,					(move_to_device(a__   ,					a__		) for t in state)		)
     elif isinstance(a__   ,					a__		):
          return type(a__		)({k: move_to_device(a__   ,					a__		) for k, v in state.items()}		)
     elif isinstance(a__   ,					torch.Tensor		):
          return state.to(a__		)
     return state







class 		lowerCAmelCase__     (   torch.optim.Optimizer      ):



 """simple docstring"""



 def __init__(	self      : Tuple      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]      ,							__SCREAMING_SNAKE_CASE      : Dict=True      ,							__SCREAMING_SNAKE_CASE      : List[Any]=None    )						->				str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   optimizer
      __SCREAMING_SNAKE_CASE						     =   scaler
      __SCREAMING_SNAKE_CASE						     =   AcceleratorState()
      __SCREAMING_SNAKE_CASE						     =   GradientState()
      __SCREAMING_SNAKE_CASE						     =   device_placement
      __SCREAMING_SNAKE_CASE						     =   False
      __SCREAMING_SNAKE_CASE						     =   None

      # Handle device placement
      if device_placement:
           __SCREAMING_SNAKE_CASE						     =   self.optimizer.state_dict()
           if self.accelerator_state.distributed_type == DistributedType.TPU:
                xm.send_cpu_data_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
           else:
                __SCREAMING_SNAKE_CASE						     =   move_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
           self.optimizer.load_state_dict(__SCREAMING_SNAKE_CASE    )
 @property
 def    UpperCAmelCase__	(	self      : str    )						->				Union[str, Any]:


      """simple docstring"""




      return self.optimizer.state
 @state.setter
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : Optional[Any]    )						->				List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   state
 @property
 def    UpperCAmelCase__	(	self      : Optional[int]    )						->				str:


      """simple docstring"""




      return self.optimizer.param_groups
 @param_groups.setter
 def    UpperCAmelCase__	(	self      : Dict      ,							__SCREAMING_SNAKE_CASE      : str    )						->				List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   param_groups
 @property
 def    UpperCAmelCase__	(	self      : List[Any]    )						->				Any:


      """simple docstring"""




      return self.optimizer.defaults
 @defaults.setter
 def    UpperCAmelCase__	(	self      : Any      ,							__SCREAMING_SNAKE_CASE      : Dict    )						->				Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE						     =   defaults
 def    UpperCAmelCase__	(	self      : Any      ,							__SCREAMING_SNAKE_CASE      : Any    )						->				List[str]:


      """simple docstring"""




      self.optimizer.add_param_group(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[str]    )						->				Dict:


      """simple docstring"""




      if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
           xm.send_cpu_data_to_device(__SCREAMING_SNAKE_CASE      ,							self.accelerator_state.device    )
      self.optimizer.load_state_dict(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : Union[str, Any]    )						->				Union[str, Any]:


      """simple docstring"""




      return self.optimizer.state_dict()
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[str]=None    )						->				Any:


      """simple docstring"""




      if self.gradient_state.sync_gradients:
           __SCREAMING_SNAKE_CASE						     =   """set_to_none""" in inspect.signature(self.optimizer.zero_grad    ).parameters
           if accept_arg:
                if set_to_none is None:
                     __SCREAMING_SNAKE_CASE						     =   False
                self.optimizer.zero_grad(set_to_none=__SCREAMING_SNAKE_CASE    )
           else:
                if set_to_none is not None:
                     raise ValueError("""`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer."""    )
                self.optimizer.zero_grad()
 def    UpperCAmelCase__	(	self      : List[str]      ,							__SCREAMING_SNAKE_CASE      : List[Any]=None    )						->				Dict:


      """simple docstring"""




      if self.gradient_state.sync_gradients:
           if self.accelerator_state.distributed_type == DistributedType.TPU:
                __SCREAMING_SNAKE_CASE						     =   {"""closure""": closure} if closure is not None else {}
                xm.optimizer_step(self.optimizer      ,							optimizer_args=__SCREAMING_SNAKE_CASE    )
           elif self.scaler is not None:
                __SCREAMING_SNAKE_CASE						     =   False
                if self._last_scale is None:
                     # `get_scale` is an async operation requiring full synchronization
                     # on CPU and GPUs before finishing. As a result, we store away
                     # the prior one to reduce the call overhead
                     __SCREAMING_SNAKE_CASE						     =   self.scaler.get_scale()
                     __SCREAMING_SNAKE_CASE						     =   True
                self.scaler.step(self.optimizer      ,							__SCREAMING_SNAKE_CASE    )
                self.scaler.update()
                __SCREAMING_SNAKE_CASE						     =   self.scaler.get_scale()
                if not new_scale:
                     # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
                     __SCREAMING_SNAKE_CASE						     =   scale_after < self._last_scale
                __SCREAMING_SNAKE_CASE						     =   scale_after

           else:
                self.optimizer.step(__SCREAMING_SNAKE_CASE    )
 def    UpperCAmelCase__	(	self      : Union[str, Any]      ,							__SCREAMING_SNAKE_CASE      : Any    )						->				Any:


      """simple docstring"""




      for param_group in self.optimizer.param_groups:
           __SCREAMING_SNAKE_CASE						     =   [parameters_map.get(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE    ) for p in param_group["""params"""]]
 @property
 def    UpperCAmelCase__	(	self      : int    )						->				Any:


      """simple docstring"""




      warnings.warn(
          """The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use """
          """`optimizer.step_was_skipped` instead."""      ,							__SCREAMING_SNAKE_CASE      ,							)
      return self._is_overflow
 @property
 def    UpperCAmelCase__	(	self      : Union[str, Any]    )						->				Optional[int]:


      """simple docstring"""




      return self._is_overflow
 def __getstate__(	self      : Tuple    )						->				str:


      """simple docstring"""




      return self.__dict__.copy()


 def __setstate__(	self      : str      ,							__SCREAMING_SNAKE_CASE      : Optional[int]    )						->				Any:


      """simple docstring"""




      self.__dict__.update(__SCREAMING_SNAKE_CASE    )






