




'''simple docstring'''




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


UpperCAmelCase_       				=      logging.get_logger(__name__)

UpperCAmelCase_       				=      {
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}

class 						lowerCAmelCase_   ( lowerCamelCase_   ):







  '''simple docstring'''







  lowerCAmelCase_				:					Optional[Any]				     =						"""bigbird_pegasus"""
  lowerCAmelCase_				:					int				     =						["""past_key_values"""]
  lowerCAmelCase_				:					Optional[Any]				     =						{
      """num_attention_heads""": """encoder_attention_heads""",
      """hidden_size""": """d_model""",
      """attention_probs_dropout_prob""": """attention_dropout""",
  }


  def __init__(						self    :	List[str]						,		_UpperCAmelCase    :	Tuple=9_61_03						,		_UpperCAmelCase    :	str=40_96						,		_UpperCAmelCase    :	Union[str, Any]=16						,		_UpperCAmelCase    :	List[str]=40_96						,		_UpperCAmelCase    :	Optional[Any]=16						,		_UpperCAmelCase    :	Tuple=16						,		_UpperCAmelCase    :	Union[str, Any]=40_96						,		_UpperCAmelCase    :	List[str]=16						,		_UpperCAmelCase    :	Optional[int]=0.0						,		_UpperCAmelCase    :	Any=0.0						,		_UpperCAmelCase    :	Any=True						,		_UpperCAmelCase    :	List[Any]=True						,		_UpperCAmelCase    :	str="gelu_new"						,		_UpperCAmelCase    :	Any=10_24						,		_UpperCAmelCase    :	Union[str, Any]=0.1						,		_UpperCAmelCase    :	List[str]=0.0						,		_UpperCAmelCase    :	List[str]=0.0						,		_UpperCAmelCase    :	int=0.02						,		_UpperCAmelCase    :	Optional[Any]=2						,		_UpperCAmelCase    :	Optional[int]=0.0						,		_UpperCAmelCase    :	Optional[int]=True						,		_UpperCAmelCase    :	List[Any]=0						,		_UpperCAmelCase    :	List[Any]=2						,		_UpperCAmelCase    :	int=1						,		_UpperCAmelCase    :	List[Any]="block_sparse"						,		_UpperCAmelCase    :	Optional[Any]=64						,		_UpperCAmelCase    :	List[str]=3						,		_UpperCAmelCase    :	Union[str, Any]=False						,		**_UpperCAmelCase    :	List[str]						,		):






   """simple docstring"""






   UpperCAmelCase__										=    vocab_size
   UpperCAmelCase__										=    max_position_embeddings
   UpperCAmelCase__										=    d_model
   UpperCAmelCase__										=    encoder_ffn_dim
   UpperCAmelCase__										=    encoder_layers
   UpperCAmelCase__										=    encoder_attention_heads
   UpperCAmelCase__										=    decoder_ffn_dim
   UpperCAmelCase__										=    decoder_layers
   UpperCAmelCase__										=    decoder_attention_heads
   UpperCAmelCase__										=    dropout
   UpperCAmelCase__										=    attention_dropout
   UpperCAmelCase__										=    activation_dropout
   UpperCAmelCase__										=    activation_function
   UpperCAmelCase__										=    init_std
   UpperCAmelCase__										=    encoder_layerdrop
   UpperCAmelCase__										=    decoder_layerdrop
   UpperCAmelCase__										=    classifier_dropout
   UpperCAmelCase__										=    use_cache
   UpperCAmelCase__										=    encoder_layers
   UpperCAmelCase__										=    scale_embedding  # scale factor will be sqrt(d_model) if True

   # extra config
   UpperCAmelCase__										=    attention_type
   UpperCAmelCase__										=    block_size
   UpperCAmelCase__										=    num_random_blocks
   UpperCAmelCase__										=    use_bias

   super().__init__(
       pad_token_id=_UpperCAmelCase						,		bos_token_id=_UpperCAmelCase						,		eos_token_id=_UpperCAmelCase						,		is_encoder_decoder=_UpperCAmelCase						,		decoder_start_token_id=_UpperCAmelCase						,		**_UpperCAmelCase						,		)

class 						lowerCAmelCase_   ( lowerCamelCase_   ):







  '''simple docstring'''







  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	str ):






   """simple docstring"""






   if self.task in ["default", "seq2seq-lm"]:
    UpperCAmelCase__										=    OrderedDict(
        [
            ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
            ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
        ] )

    if self.use_past:
     UpperCAmelCase__										=    {0: """batch"""}
     UpperCAmelCase__										=    {0: """batch""", 1: """past_decoder_sequence + sequence"""}
    else:
     UpperCAmelCase__										=    {0: """batch""", 1: """decoder_sequence"""}
     UpperCAmelCase__										=    {0: """batch""", 1: """decoder_sequence"""}

    if self.use_past:
     self.fill_with_past_key_values_(_UpperCAmelCase						,		direction="""inputs""" )
   elif self.task == "causal-lm":
    # TODO: figure this case out.
    UpperCAmelCase__										=    OrderedDict(
        [
            ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
            ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
        ] )
    if self.use_past:
     UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_layers
     for i in range(_UpperCAmelCase ):
      UpperCAmelCase__										=    {0: """batch""", 2: """past_sequence + sequence"""}
      UpperCAmelCase__										=    {0: """batch""", 2: """past_sequence + sequence"""}
   else:
    UpperCAmelCase__										=    OrderedDict(
        [
            ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
            ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
            ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
            ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
        ] )

   return common_inputs


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Dict ):






   """simple docstring"""






   if self.task in ["default", "seq2seq-lm"]:
    UpperCAmelCase__										=    super().outputs
   else:
    UpperCAmelCase__										=    super(_UpperCAmelCase						,		self ).outputs
    if self.use_past:
     UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_layers
     for i in range(_UpperCAmelCase ):
      UpperCAmelCase__										=    {0: """batch""", 2: """past_sequence + sequence"""}
      UpperCAmelCase__										=    {0: """batch""", 2: """past_sequence + sequence"""}
   return common_outputs


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int]						,		_UpperCAmelCase    :	PreTrainedTokenizer						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	bool = False						,		_UpperCAmelCase    :	Optional[TensorType] = None						,		):






   """simple docstring"""






   UpperCAmelCase__										=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       _UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase )

   # Generate decoder inputs
   UpperCAmelCase__										=    seq_length if not self.use_past else 1
   UpperCAmelCase__										=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       _UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase )
   UpperCAmelCase__										=    {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
   UpperCAmelCase__										=    dict(**_UpperCAmelCase						,		**_UpperCAmelCase )

   if self.use_past:
    if not is_torch_available():
     raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed.""" )
    else:
     import torch
    UpperCAmelCase__							,			UpperCAmelCase__										=    common_inputs["""input_ids"""].shape
    UpperCAmelCase__										=    common_inputs["""decoder_input_ids"""].shape[1]
    UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_attention_heads
    UpperCAmelCase__										=    (
        batch,
        num_encoder_attention_heads,
        encoder_seq_length,
        self._config.hidden_size // num_encoder_attention_heads,
    )
    UpperCAmelCase__										=    decoder_seq_length + 3
    UpperCAmelCase__										=    (
        batch,
        num_decoder_attention_heads,
        decoder_past_length,
        self._config.hidden_size // num_decoder_attention_heads,
    )

    UpperCAmelCase__										=    torch.cat(
        [common_inputs["""decoder_attention_mask"""], torch.ones(_UpperCAmelCase						,		_UpperCAmelCase )]						,		dim=1 )

    UpperCAmelCase__										=    []
    # If the number of encoder and decoder layers are present in the model configuration, both are considered
    UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_layers
    UpperCAmelCase__										=    min(_UpperCAmelCase						,		_UpperCAmelCase )
    UpperCAmelCase__										=    max(_UpperCAmelCase						,		_UpperCAmelCase ) - min_num_layers
    UpperCAmelCase__										=    """encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

    for _ in range(_UpperCAmelCase ):
     common_inputs["past_key_values"].append(
         (
             torch.zeros(_UpperCAmelCase ),
             torch.zeros(_UpperCAmelCase ),
             torch.zeros(_UpperCAmelCase ),
             torch.zeros(_UpperCAmelCase ),
         ) )
    # TODO: test this.
    UpperCAmelCase__										=    encoder_shape if remaining_side_name == """encoder""" else decoder_shape
    for _ in range(_UpperCAmelCase						,		_UpperCAmelCase ):
     common_inputs["past_key_values"].append((torch.zeros(_UpperCAmelCase ), torch.zeros(_UpperCAmelCase )) )
   return common_inputs


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Union[str, Any]						,		_UpperCAmelCase    :	PreTrainedTokenizer						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	bool = False						,		_UpperCAmelCase    :	Optional[TensorType] = None						,		):






   """simple docstring"""






   UpperCAmelCase__										=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       _UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase )

   if self.use_past:
    if not is_torch_available():
     raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed.""" )
    else:
     import torch
    UpperCAmelCase__							,			UpperCAmelCase__										=    common_inputs["""input_ids"""].shape
    # Not using the same length for past_key_values
    UpperCAmelCase__										=    seqlen + 2
    UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_layers
    UpperCAmelCase__							,			UpperCAmelCase__										=    self.num_attention_heads
    UpperCAmelCase__										=    (
        batch,
        num_encoder_attention_heads,
        past_key_values_length,
        self._config.hidden_size // num_encoder_attention_heads,
    )

    UpperCAmelCase__										=    common_inputs["""attention_mask"""].dtype
    UpperCAmelCase__										=    torch.cat(
        [common_inputs["""attention_mask"""], torch.ones(_UpperCAmelCase						,		_UpperCAmelCase						,		dtype=_UpperCAmelCase )]						,		dim=1 )
    UpperCAmelCase__										=    [
        (torch.zeros(_UpperCAmelCase ), torch.zeros(_UpperCAmelCase )) for _ in range(_UpperCAmelCase )
    ]
   return common_inputs


  def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	PreTrainedTokenizer						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	bool = False						,		_UpperCAmelCase    :	Optional[TensorType] = None						,		):






   """simple docstring"""






   UpperCAmelCase__										=    compute_effective_axis_dimension(
       _UpperCAmelCase						,		fixed_dimension=OnnxConfig.default_fixed_batch						,		num_token_to_add=0 )

   # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
   UpperCAmelCase__										=    tokenizer.num_special_tokens_to_add(_UpperCAmelCase )
   UpperCAmelCase__										=    compute_effective_axis_dimension(
       _UpperCAmelCase						,		fixed_dimension=OnnxConfig.default_fixed_sequence						,		num_token_to_add=_UpperCAmelCase )

   # Generate dummy inputs according to compute batch and sequence
   UpperCAmelCase__										=    [""" """.join([tokenizer.unk_token] ) * seq_length] * batch_size
   UpperCAmelCase__										=    dict(tokenizer(_UpperCAmelCase						,		return_tensors=_UpperCAmelCase ) )
   return common_inputs


  def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	PreTrainedTokenizer						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	int = -1						,		_UpperCAmelCase    :	bool = False						,		_UpperCAmelCase    :	Optional[TensorType] = None						,		):






   """simple docstring"""






   if self.task in ["default", "seq2seq-lm"]:
    UpperCAmelCase__										=    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
        _UpperCAmelCase						,		batch_size=_UpperCAmelCase						,		seq_length=_UpperCAmelCase						,		is_pair=_UpperCAmelCase						,		framework=_UpperCAmelCase )

   elif self.task == "causal-lm":
    UpperCAmelCase__										=    self._generate_dummy_inputs_for_causal_lm(
        _UpperCAmelCase						,		batch_size=_UpperCAmelCase						,		seq_length=_UpperCAmelCase						,		is_pair=_UpperCAmelCase						,		framework=_UpperCAmelCase )
   else:
    UpperCAmelCase__										=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
        _UpperCAmelCase						,		batch_size=_UpperCAmelCase						,		seq_length=_UpperCAmelCase						,		is_pair=_UpperCAmelCase						,		framework=_UpperCAmelCase )

   return common_inputs



  def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	List[Any]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Tuple						,		_UpperCAmelCase    :	List[Any] ):






   """simple docstring"""






   if self.task in ["default", "seq2seq-lm"]:
    UpperCAmelCase__										=    super()._flatten_past_key_values_(_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase )
   else:
    UpperCAmelCase__										=    super(_UpperCAmelCase						,		self )._flatten_past_key_values_(
        _UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase						,		_UpperCAmelCase )






'''simple docstring'''




import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm

def    _UpperCamelCase      (				SCREAMING_SNAKE_CASE__			:      Tuple       ,						SCREAMING_SNAKE_CASE__			:      Dict       ):
 '''simple docstring'''





 if isinstance(SCREAMING_SNAKE_CASE__       ,						(list, tuple)       ):
  return honor_type(SCREAMING_SNAKE_CASE__       ,						(move_to_device(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ) for t in state)       )
 elif isinstance(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ):
  return type(SCREAMING_SNAKE_CASE__       )({k: move_to_device(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ) for k, v in state.items()}       )
 elif isinstance(SCREAMING_SNAKE_CASE__       ,						torch.Tensor       ):
  return state.to(SCREAMING_SNAKE_CASE__       )
 return state







class 						lowerCAmelCase_   ( torch.optim.Optimizer   ):







  '''simple docstring'''







  def __init__(						self    :	List[str]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Union[str, Any]=True						,		_UpperCAmelCase    :	List[str]=None ):






   """simple docstring"""






   UpperCAmelCase__										=    optimizer
   UpperCAmelCase__										=    scaler
   UpperCAmelCase__										=    AcceleratorState()
   UpperCAmelCase__										=    GradientState()
   UpperCAmelCase__										=    device_placement
   UpperCAmelCase__										=    False
   UpperCAmelCase__										=    None

   # Handle device placement
   if device_placement:
    UpperCAmelCase__										=    self.optimizer.state_dict()
    if self.accelerator_state.distributed_type == DistributedType.TPU:
     xm.send_cpu_data_to_device(_UpperCAmelCase						,		self.accelerator_state.device )
    else:
     UpperCAmelCase__										=    move_to_device(_UpperCAmelCase						,		self.accelerator_state.device )
    self.optimizer.load_state_dict(_UpperCAmelCase )


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Dict ):






   """simple docstring"""






   return self.optimizer.state


  @state.setter
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int]						,		_UpperCAmelCase    :	List[str] ):






   """simple docstring"""






   UpperCAmelCase__										=    state


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Any ):






   """simple docstring"""






   return self.optimizer.param_groups


  @param_groups.setter
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int]						,		_UpperCAmelCase    :	List[Any] ):






   """simple docstring"""






   UpperCAmelCase__										=    param_groups


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[Any] ):






   """simple docstring"""






   return self.optimizer.defaults


  @defaults.setter
  def 							SCREAMING_SNAKE_CASE__			(						self    :	int						,		_UpperCAmelCase    :	str ):






   """simple docstring"""






   UpperCAmelCase__										=    defaults


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int]						,		_UpperCAmelCase    :	Union[str, Any] ):






   """simple docstring"""






   self.optimizer.add_param_group(_UpperCAmelCase )


  def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	Tuple ):






   """simple docstring"""






   if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
    xm.send_cpu_data_to_device(_UpperCAmelCase						,		self.accelerator_state.device )
   self.optimizer.load_state_dict(_UpperCAmelCase )


  def 							SCREAMING_SNAKE_CASE__			(						self    :	str ):






   """simple docstring"""






   return self.optimizer.state_dict()


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int]						,		_UpperCAmelCase    :	List[Any]=None ):






   """simple docstring"""






   if self.gradient_state.sync_gradients:
    UpperCAmelCase__										=    """set_to_none""" in inspect.signature(self.optimizer.zero_grad ).parameters
    if accept_arg:
     if set_to_none is None:
      UpperCAmelCase__										=    False
     self.optimizer.zero_grad(set_to_none=_UpperCAmelCase )
    else:
     if set_to_none is not None:
      raise ValueError("""`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.""" )
     self.optimizer.zero_grad()


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Any						,		_UpperCAmelCase    :	List[str]=None ):






   """simple docstring"""






   if self.gradient_state.sync_gradients:
    if self.accelerator_state.distributed_type == DistributedType.TPU:
     UpperCAmelCase__										=    {"""closure""": closure} if closure is not None else {}
     xm.optimizer_step(self.optimizer						,		optimizer_args=_UpperCAmelCase )
    elif self.scaler is not None:
     UpperCAmelCase__										=    False
     if self._last_scale is None:
      # `get_scale` is an async operation requiring full synchronization
      # on CPU and GPUs before finishing. As a result, we store away
      # the prior one to reduce the call overhead
      UpperCAmelCase__										=    self.scaler.get_scale()
      UpperCAmelCase__										=    True
     self.scaler.step(self.optimizer						,		_UpperCAmelCase )
     self.scaler.update()
     UpperCAmelCase__										=    self.scaler.get_scale()
     if not new_scale:
      # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
      UpperCAmelCase__										=    scale_after < self._last_scale
     UpperCAmelCase__										=    scale_after

    else:
     self.optimizer.step(_UpperCAmelCase )


  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[Any]						,		_UpperCAmelCase    :	str ):






   """simple docstring"""






   for param_group in self.optimizer.param_groups:
    UpperCAmelCase__										=    [parameters_map.get(_UpperCAmelCase						,		_UpperCAmelCase ) for p in param_group["""params"""]]


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	str ):






   """simple docstring"""






   warnings.warn(
       """The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use """
       """`optimizer.step_was_skipped` instead."""						,		_UpperCAmelCase						,		)
   return self._is_overflow


  @property
  def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int] ):






   """simple docstring"""






   return self._is_overflow


  def __getstate__(						self    :	Dict ):






   """simple docstring"""






   return self.__dict__.copy()



  def __setstate__(						self    :	Dict						,		_UpperCAmelCase    :	Tuple ):






   """simple docstring"""






   self.__dict__.update(_UpperCAmelCase )
