






import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
		import jax
		import jax.numpy as jnp

		from transformers.modeling_flax_pytorch_utils import (
		    convert_pytorch_state_dict_to_flax,
		    load_flax_weights_in_pytorch_model,
		)
		from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
		import torch





class __lowerCamelCase				:





	"""simple docstring"""






	def __init__(							self ,					UpperCAmelCase ,					UpperCAmelCase=14 ,					UpperCAmelCase=7 ,					UpperCAmelCase=True ,					UpperCAmelCase=True ,					UpperCAmelCase=False ,					UpperCAmelCase=True ,					UpperCAmelCase=99 ,					UpperCAmelCase=32 ,					UpperCAmelCase=5 ,					UpperCAmelCase=4 ,					UpperCAmelCase=37 ,					UpperCAmelCase="gelu" ,					UpperCAmelCase=0.1 ,					UpperCAmelCase=0.1 ,					UpperCAmelCase=512 ,					UpperCAmelCase=0.02 ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    parent
							_UpperCAmelCase             =    batch_size
							_UpperCAmelCase             =    seq_length
							_UpperCAmelCase             =    is_training
							_UpperCAmelCase             =    use_input_mask
							_UpperCAmelCase             =    use_token_type_ids
							_UpperCAmelCase             =    use_labels
							_UpperCAmelCase             =    vocab_size
							_UpperCAmelCase             =    hidden_size
							_UpperCAmelCase             =    num_hidden_layers
							_UpperCAmelCase             =    num_attention_heads
							_UpperCAmelCase             =    intermediate_size
							_UpperCAmelCase             =    hidden_act
							_UpperCAmelCase             =    hidden_dropout_prob
							_UpperCAmelCase             =    attention_probs_dropout_prob
							_UpperCAmelCase             =    max_position_embeddings
							_UpperCAmelCase             =    initializer_range
							_UpperCAmelCase             =    None
							_UpperCAmelCase             =    vocab_size - 1
							_UpperCAmelCase             =    vocab_size - 1
							_UpperCAmelCase             =    vocab_size - 1

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    ids_tensor([self.batch_size, self.seq_length] ,					self.vocab_size					)

							_UpperCAmelCase             =    None
							if self.use_input_mask:
													_UpperCAmelCase             =    random_attention_mask([self.batch_size, self.seq_length]					)

							_UpperCAmelCase             =    GPTaConfig(
							    vocab_size=self.vocab_size ,					n_embd=self.hidden_size ,					n_layer=self.num_hidden_layers ,					n_head=self.num_attention_heads ,					n_positions=self.max_position_embeddings ,					use_cache=UpperCAmelCase ,					bos_token_id=self.bos_token_id ,					eos_token_id=self.eos_token_id ,					pad_token_id=self.pad_token_id ,					)

							return (config, input_ids, input_mask)

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.prepare_config_and_inputs()
							_UpperCAmelCase      ,				_UpperCAmelCase      ,				_UpperCAmelCase             =    config_and_inputs
							_UpperCAmelCase             =    {'input_ids': input_ids, 'attention_mask': attention_mask}
							return config, inputs_dict

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase      ,				_UpperCAmelCase      ,				_UpperCAmelCase             =    self.prepare_config_and_inputs()

							_UpperCAmelCase             =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]					)
							_UpperCAmelCase             =    ids_tensor([self.batch_size, self.seq_length] ,					vocab_size=2					)

							return (
							    config,
							    input_ids,
							    attention_mask,
							    encoder_hidden_states,
							    encoder_attention_mask,
							)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    20
							_UpperCAmelCase             =    model_class_name(UpperCAmelCase					)

							_UpperCAmelCase             =    model.init_cache(input_ids.shape[0] ,					UpperCAmelCase					)
							_UpperCAmelCase             =    jnp.ones((input_ids.shape[0], max_decoder_length) ,					dtype='i4'					)

							_UpperCAmelCase             =    jnp.broadcast_to(
							    jnp.arange(input_ids.shape[-1] - 1					)[None, :] ,					(input_ids.shape[0], input_ids.shape[-1] - 1)					)
							_UpperCAmelCase             =    model(
							    input_ids[:, :-1] ,					attention_mask=UpperCAmelCase ,					past_key_values=UpperCAmelCase ,					position_ids=UpperCAmelCase ,					)

							_UpperCAmelCase             =    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,					dtype='i4'					)
							_UpperCAmelCase             =    model(
							    input_ids[:, -1:] ,					attention_mask=UpperCAmelCase ,					past_key_values=outputs_cache.past_key_values ,					position_ids=UpperCAmelCase ,					)

							_UpperCAmelCase             =    model(UpperCAmelCase					)

							_UpperCAmelCase             =    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])					)					)
							self.parent.assertTrue(diff < 1e-3 ,					msg=F"""Max diff is {diff}"""					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    20
							_UpperCAmelCase             =    model_class_name(UpperCAmelCase					)

							_UpperCAmelCase             =    jnp.concatenate(
							    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])					)] ,					axis=-1 ,					)

							_UpperCAmelCase             =    model.init_cache(input_ids.shape[0] ,					UpperCAmelCase					)
							_UpperCAmelCase             =    jnp.broadcast_to(
							    jnp.arange(input_ids.shape[-1] - 1					)[None, :] ,					(input_ids.shape[0], input_ids.shape[-1] - 1)					)

							_UpperCAmelCase             =    model(
							    input_ids[:, :-1] ,					attention_mask=UpperCAmelCase ,					past_key_values=UpperCAmelCase ,					position_ids=UpperCAmelCase ,					)
							_UpperCAmelCase             =    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,					dtype='i4'					)
							_UpperCAmelCase             =    model(
							    input_ids[:, -1:] ,					past_key_values=outputs_cache.past_key_values ,					attention_mask=UpperCAmelCase ,					position_ids=UpperCAmelCase ,					)

							_UpperCAmelCase             =    model(UpperCAmelCase ,					attention_mask=UpperCAmelCase					)

							_UpperCAmelCase             =    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])					)					)
							self.parent.assertTrue(diff < 1e-3 ,					msg=F"""Max diff is {diff}"""					)





@require_flax
class __lowerCamelCase				(   snake_case__  , snake_case__  , unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__			=   (FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
	UpperCamelCase__			=   (FlaxGPTaLMHeadModel,) if is_flax_available() else ()

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    FlaxGPTaModelTester(self					)

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							for model_class_name in self.all_model_classes:
													_UpperCAmelCase      ,				_UpperCAmelCase      ,				_UpperCAmelCase             =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.check_use_cache_forward(UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					)

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							for model_class_name in self.all_model_classes:
													_UpperCAmelCase      ,				_UpperCAmelCase      ,				_UpperCAmelCase             =    self.model_tester.prepare_config_and_inputs()
													self.model_tester.check_use_cache_forward_with_attn_mask(
													    UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					)

	@slow
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    GPTaTokenizer.from_pretrained('gpt2' ,					pad_token='</s>' ,					padding_side='left'					)
							_UpperCAmelCase             =    tokenizer(['Hello this is a long string', 'Hey'] ,					return_tensors='np' ,					padding=UpperCAmelCase ,					truncation=UpperCAmelCase					)

							_UpperCAmelCase             =    FlaxGPTaLMHeadModel.from_pretrained('gpt2'					)
							_UpperCAmelCase             =    False
							_UpperCAmelCase             =    model.config.eos_token_id

							_UpperCAmelCase             =    jax.jit(model.generate					)

							_UpperCAmelCase             =    jit_generate(inputs['input_ids'] ,					attention_mask=inputs['attention_mask']					).sequences

							_UpperCAmelCase             =    tokenizer.batch_decode(UpperCAmelCase ,					skip_special_tokens=UpperCAmelCase					)

							_UpperCAmelCase             =    [
							    'Hello this is a long string of words. I\'m going to start with the first one.\n',
							    'Hey, I\'m not sure if I\'m going to be able to do',
							]

							self.assertListEqual(UpperCAmelCase ,					UpperCAmelCase					)

	@is_pt_flax_cross_test
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase      ,				_UpperCAmelCase             =    self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													with self.subTest(model_class.__name__					):
																			# prepare inputs
																			_UpperCAmelCase             =    self._prepare_for_class(UpperCAmelCase ,					UpperCAmelCase					)
																			_UpperCAmelCase             =    {k: torch.tensor(v.tolist()					) for k, v in prepared_inputs_dict.items()}

																			# load corresponding PyTorch class
																			_UpperCAmelCase             =    model_class.__name__[4:]  # Skip the "Flax" at the beginning
																			_UpperCAmelCase             =    getattr(UpperCAmelCase ,					UpperCAmelCase					)

																			_UpperCAmelCase      ,				_UpperCAmelCase             =    pt_inputs['input_ids'].shape
																			_UpperCAmelCase             =    np.random.randint(0 ,					seq_length - 1 ,					size=(batch_size,)					)
																			for batch_idx, start_index in enumerate(UpperCAmelCase					):
																									_UpperCAmelCase             =    0
																									_UpperCAmelCase             =    1
																									_UpperCAmelCase             =    0
																									_UpperCAmelCase             =    1
																			_UpperCAmelCase             =    pt_model_class(UpperCAmelCase					).eval()
																			_UpperCAmelCase             =    model_class(UpperCAmelCase ,					dtype=jnp.floataa					)

																			_UpperCAmelCase             =    convert_pytorch_state_dict_to_flax(pt_model.state_dict() ,					UpperCAmelCase					)
																			_UpperCAmelCase             =    fx_state

																			with torch.no_grad():
																									_UpperCAmelCase             =    pt_model(**UpperCAmelCase					).to_tuple()

																			_UpperCAmelCase             =    fx_model(**UpperCAmelCase					).to_tuple()
																			self.assertEqual(len(UpperCAmelCase					) ,					len(UpperCAmelCase					) ,					'Output lengths differ between Flax and PyTorch'					)
																			for fx_output, pt_output in zip(UpperCAmelCase ,					UpperCAmelCase					):
																									self.assert_almost_equals(fx_output[:, -1] ,					pt_output[:, -1].numpy() ,					4e-2					)

																			with tempfile.TemporaryDirectory() as tmpdirname:
																									pt_model.save_pretrained(UpperCAmelCase					)
																									_UpperCAmelCase             =    model_class.from_pretrained(UpperCAmelCase ,					from_pt=UpperCAmelCase					)

																			_UpperCAmelCase             =    fx_model_loaded(**UpperCAmelCase					).to_tuple()
																			self.assertEqual(
																			    len(UpperCAmelCase					) ,					len(UpperCAmelCase					) ,					'Output lengths differ between Flax and PyTorch'					)
																			for fx_output_loaded, pt_output in zip(UpperCAmelCase ,					UpperCAmelCase					):
																									self.assert_almost_equals(fx_output_loaded[:, -1] ,					pt_output[:, -1].numpy() ,					4e-2					)

	@is_pt_flax_cross_test
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase      ,				_UpperCAmelCase             =    self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
													with self.subTest(model_class.__name__					):
																			# prepare inputs
																			_UpperCAmelCase             =    self._prepare_for_class(UpperCAmelCase ,					UpperCAmelCase					)
																			_UpperCAmelCase             =    {k: torch.tensor(v.tolist()					) for k, v in prepared_inputs_dict.items()}

																			# load corresponding PyTorch class
																			_UpperCAmelCase             =    model_class.__name__[4:]  # Skip the "Flax" at the beginning
																			_UpperCAmelCase             =    getattr(UpperCAmelCase ,					UpperCAmelCase					)

																			_UpperCAmelCase             =    pt_model_class(UpperCAmelCase					).eval()
																			_UpperCAmelCase             =    model_class(UpperCAmelCase ,					dtype=jnp.floataa					)

																			_UpperCAmelCase             =    load_flax_weights_in_pytorch_model(UpperCAmelCase ,					fx_model.params					)
																			_UpperCAmelCase      ,				_UpperCAmelCase             =    pt_inputs['input_ids'].shape
																			_UpperCAmelCase             =    np.random.randint(0 ,					seq_length - 1 ,					size=(batch_size,)					)
																			for batch_idx, start_index in enumerate(UpperCAmelCase					):
																									_UpperCAmelCase             =    0
																									_UpperCAmelCase             =    1
																									_UpperCAmelCase             =    0
																									_UpperCAmelCase             =    1

																			# make sure weights are tied in PyTorch
																			pt_model.tie_weights()

																			with torch.no_grad():
																									_UpperCAmelCase             =    pt_model(**UpperCAmelCase					).to_tuple()

																			_UpperCAmelCase             =    fx_model(**UpperCAmelCase					).to_tuple()
																			self.assertEqual(len(UpperCAmelCase					) ,					len(UpperCAmelCase					) ,					'Output lengths differ between Flax and PyTorch'					)
																			for fx_output, pt_output in zip(UpperCAmelCase ,					UpperCAmelCase					):
																									self.assert_almost_equals(fx_output[:, -1] ,					pt_output[:, -1].numpy() ,					4e-2					)

																			with tempfile.TemporaryDirectory() as tmpdirname:
																									fx_model.save_pretrained(UpperCAmelCase					)
																									_UpperCAmelCase             =    pt_model_class.from_pretrained(UpperCAmelCase ,					from_flax=UpperCAmelCase					)

																			with torch.no_grad():
																									_UpperCAmelCase             =    pt_model_loaded(**UpperCAmelCase					).to_tuple()

																			self.assertEqual(
																			    len(UpperCAmelCase					) ,					len(UpperCAmelCase					) ,					'Output lengths differ between Flax and PyTorch'					)
																			for fx_output, pt_output in zip(UpperCAmelCase ,					UpperCAmelCase					):
																									self.assert_almost_equals(fx_output[:, -1] ,					pt_output[:, -1].numpy() ,					4e-2					)

	@slow
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							for model_class_name in self.all_model_classes:
													_UpperCAmelCase             =    model_class_name.from_pretrained('gpt2' ,					from_pt=UpperCAmelCase					)
													_UpperCAmelCase             =    model(np.ones((1, 1)					)					)
													self.assertIsNotNone(UpperCAmelCase					)








from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_a   			=						logging.get_logger(__name__)

_a   			=						{
    '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json''',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	UpperCamelCase__			=   "m2m_100"
	UpperCamelCase__			=   ["past_key_values"]
	UpperCamelCase__			=   {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

	def __init__(							self ,					UpperCAmelCase=12_8112 ,					UpperCAmelCase=1024 ,					UpperCAmelCase=12 ,					UpperCAmelCase=4096 ,					UpperCAmelCase=16 ,					UpperCAmelCase=12 ,					UpperCAmelCase=4096 ,					UpperCAmelCase=16 ,					UpperCAmelCase=0.05 ,					UpperCAmelCase=0.05 ,					UpperCAmelCase=True ,					UpperCAmelCase=True ,					UpperCAmelCase="relu" ,					UpperCAmelCase=1024 ,					UpperCAmelCase=0.1 ,					UpperCAmelCase=0.1 ,					UpperCAmelCase=0.0 ,					UpperCAmelCase=0.02 ,					UpperCAmelCase=2 ,					UpperCAmelCase=True ,					UpperCAmelCase=1 ,					UpperCAmelCase=0 ,					UpperCAmelCase=2 ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    vocab_size
							_UpperCAmelCase             =    max_position_embeddings
							_UpperCAmelCase             =    d_model
							_UpperCAmelCase             =    encoder_ffn_dim
							_UpperCAmelCase             =    encoder_layers
							_UpperCAmelCase             =    encoder_attention_heads
							_UpperCAmelCase             =    decoder_ffn_dim
							_UpperCAmelCase             =    decoder_layers
							_UpperCAmelCase             =    decoder_attention_heads
							_UpperCAmelCase             =    dropout
							_UpperCAmelCase             =    attention_dropout
							_UpperCAmelCase             =    activation_dropout
							_UpperCAmelCase             =    activation_function
							_UpperCAmelCase             =    init_std
							_UpperCAmelCase             =    encoder_layerdrop
							_UpperCAmelCase             =    decoder_layerdrop
							_UpperCAmelCase             =    use_cache
							_UpperCAmelCase             =    encoder_layers
							_UpperCAmelCase             =    scale_embedding  # scale factor will be sqrt(d_model) if True

							super().__init__(
							    pad_token_id=UpperCAmelCase ,					bos_token_id=UpperCAmelCase ,					eos_token_id=UpperCAmelCase ,					is_encoder_decoder=UpperCAmelCase ,					decoder_start_token_id=UpperCAmelCase ,					**UpperCAmelCase ,					)





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	@property
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    OrderedDict(
							    [
							        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
							        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
							    ]					)

							if self.use_past:
													_UpperCAmelCase             =    {0: 'batch'}
													_UpperCAmelCase             =    {0: 'batch', 1: 'past_decoder_sequence + sequence'}
							else:
													_UpperCAmelCase             =    {0: 'batch', 1: 'decoder_sequence'}
													_UpperCAmelCase             =    {0: 'batch', 1: 'decoder_sequence'}

							if self.use_past:
													self.fill_with_past_key_values_(UpperCAmelCase ,					direction='inputs'					)
							return common_inputs

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = -1 ,					UpperCAmelCase = -1 ,					UpperCAmelCase = False ,					UpperCAmelCase = None ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    compute_effective_axis_dimension(
							    UpperCAmelCase ,					fixed_dimension=OnnxConfig.default_fixed_batch ,					num_token_to_add=0					)

							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							_UpperCAmelCase             =    tokenizer.num_special_tokens_to_add(UpperCAmelCase					)
							_UpperCAmelCase             =    compute_effective_axis_dimension(
							    UpperCAmelCase ,					fixed_dimension=OnnxConfig.default_fixed_sequence ,					num_token_to_add=UpperCAmelCase					)

							# Generate dummy inputs according to compute batch and sequence
							_UpperCAmelCase             =    [' '.join([tokenizer.unk_token]					) * seq_length] * batch_size
							_UpperCAmelCase             =    dict(tokenizer(UpperCAmelCase ,					return_tensors=UpperCAmelCase					)					)
							return common_inputs

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = -1 ,					UpperCAmelCase = -1 ,					UpperCAmelCase = False ,					UpperCAmelCase = None ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					)

							# Generate decoder inputs
							_UpperCAmelCase             =    seq_length if not self.use_past else 1
							_UpperCAmelCase             =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase					)
							_UpperCAmelCase             =    {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
							_UpperCAmelCase             =    dict(**UpperCAmelCase ,					**UpperCAmelCase					)

							if self.use_past:
													if not is_torch_available():
																			raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'					)
													else:
																			import torch
													_UpperCAmelCase      ,				_UpperCAmelCase             =    common_inputs['input_ids'].shape
													_UpperCAmelCase             =    common_inputs['decoder_input_ids'].shape[1]
													_UpperCAmelCase      ,				_UpperCAmelCase             =    self.num_attention_heads
													_UpperCAmelCase             =    (
													    batch,
													    num_encoder_attention_heads,
													    encoder_seq_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)
													_UpperCAmelCase             =    decoder_seq_length + 3
													_UpperCAmelCase             =    (
													    batch,
													    num_decoder_attention_heads,
													    decoder_past_length,
													    self._config.hidden_size // num_decoder_attention_heads,
													)

													_UpperCAmelCase             =    torch.cat(
													    [common_inputs['decoder_attention_mask'], torch.ones(UpperCAmelCase ,					UpperCAmelCase					)] ,					dim=1					)

													_UpperCAmelCase             =    []
													# If the number of encoder and decoder layers are present in the model configuration, both are considered
													_UpperCAmelCase      ,				_UpperCAmelCase             =    self.num_layers
													_UpperCAmelCase             =    min(UpperCAmelCase ,					UpperCAmelCase					)
													_UpperCAmelCase             =    max(UpperCAmelCase ,					UpperCAmelCase					) - min_num_layers
													_UpperCAmelCase             =    'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

													for _ in range(UpperCAmelCase					):
																			common_inputs["past_key_values"].append(
																			    (
																			        torch.zeros(UpperCAmelCase					),
																			        torch.zeros(UpperCAmelCase					),
																			        torch.zeros(UpperCAmelCase					),
																			        torch.zeros(UpperCAmelCase					),
																			    )					)
													# TODO: test this.
													_UpperCAmelCase             =    encoder_shape if remaining_side_name == 'encoder' else decoder_shape
													for _ in range(UpperCAmelCase ,					UpperCAmelCase					):
																			common_inputs["past_key_values"].append((torch.zeros(UpperCAmelCase					), torch.zeros(UpperCAmelCase					))					)
							return common_inputs

	UpperCamelCase__			=   _generate_dummy_inputs_for_default_and_seqaseq_lm
