


import argparse
import logging
import os
import sys
import tempfile
from pathlib import Path

import lightning_base
import pytest
import pytorch_lightning as pl
import torch
from convert_pl_checkpoint_to_hf import convert_pl_to_hf
from distillation import distill_main
from finetune import SummarizationModule, main
from huggingface_hub import list_models
from parameterized import parameterized
from run_eval import generate_summaries_or_translations
from torch import nn

from transformers import AutoConfig, AutoModelForSeqaSeqLM
from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
from utils import label_smoothed_nll_loss, lmap, load_json


logging.basicConfig(level=logging.DEBUG)

UpperCAmelCase__     			=							logging.getLogger()
UpperCAmelCase__     			=							torch.cuda.is_available()
UpperCAmelCase__     			=							{
    '''max_tokens_per_batch''': None,
    '''supervise_forward''': True,
    '''normalize_hidden''': True,
    '''label_smoothing''': 0.2,
    '''eval_max_gen_length''': None,
    '''eval_beams''': 1,
    '''val_metric''': '''loss''',
    '''save_top_k''': 1,
    '''adafactor''': True,
    '''early_stopping_patience''': 2,
    '''logger_name''': '''default''',
    '''length_penalty''': 0.5,
    '''cache_dir''': '''''',
    '''task''': '''summarization''',
    '''num_workers''': 2,
    '''alpha_hid''': 0,
    '''freeze_embeds''': True,
    '''enc_only''': False,
    '''tgt_suffix''': '''''',
    '''resume_from_checkpoint''': None,
    '''sortish_sampler''': True,
    '''student_decoder_layers''': 1,
    '''val_check_interval''': 1.0,
    '''output_dir''': '''''',
    '''fp16''': False,  # TODO(SS): set this to CUDA_AVAILABLE if ci installs apex or start using native amp
    '''no_teacher''': False,
    '''fp16_opt_level''': '''O1''',
    '''gpus''': 1 if CUDA_AVAILABLE else 0,
    '''n_tpu_cores''': 0,
    '''max_grad_norm''': 1.0,
    '''do_train''': True,
    '''do_predict''': True,
    '''accumulate_grad_batches''': 1,
    '''server_ip''': '''''',
    '''server_port''': '''''',
    '''seed''': 42,
    '''model_name_or_path''': '''sshleifer/bart-tiny-random''',
    '''config_name''': '''''',
    '''tokenizer_name''': '''facebook/bart-large''',
    '''do_lower_case''': False,
    '''learning_rate''': 0.3,
    '''lr_scheduler''': '''linear''',
    '''weight_decay''': 0.0,
    '''adam_epsilon''': 1e-08,
    '''warmup_steps''': 0,
    '''max_epochs''': 1,
    '''train_batch_size''': 2,
    '''eval_batch_size''': 2,
    '''max_source_length''': 12,
    '''max_target_length''': 12,
    '''val_max_target_length''': 12,
    '''test_max_target_length''': 12,
    '''fast_dev_run''': False,
    '''no_cache''': False,
    '''n_train''': -1,
    '''n_val''': -1,
    '''n_test''': -1,
    '''student_encoder_layers''': 1,
    '''freeze_encoder''': False,
    '''auto_scale_batch_size''': False,
    '''overwrite_output_dir''': False,
    '''student''': None,
}
def        UpperCAmelCase_      (							__snake_case  ,		__snake_case		)  ->    Optional[int]:







 """simple docstring"""


 _lowercase							   ='''\n'''.join(__snake_case		)
 Path(__snake_case		).open('''w'''		).writelines(__snake_case		)


UpperCAmelCase__     			=							[''' Sam ate lunch today.''', '''Sams lunch ingredients.''']
UpperCAmelCase__     			=							['''A very interesting story about what I ate for lunch.''', '''Avocado, celery, turkey, coffee''']
UpperCAmelCase__     			=							'''patrickvonplaten/t5-tiny-random'''
UpperCAmelCase__     			=							'''sshleifer/t5-tinier-random'''
UpperCAmelCase__     			=							'''sshleifer/bart-tiny-random'''
UpperCAmelCase__     			=							'''sshleifer/tiny-mbart'''
UpperCAmelCase__     			=							'''sshleifer/tiny-marian-en-de'''
UpperCAmelCase__     			=							'''stas/tiny-wmt19-en-de'''


UpperCAmelCase__     			=							logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
def        UpperCAmelCase_      (							__snake_case		)  ->    Optional[int]:







 """simple docstring"""


 for split in ["train", "val", "test"]:
  _dump_articles(os.path.join(__snake_case  ,		F"{split}.source"		)  ,		__snake_case		)
  _dump_articles(os.path.join(__snake_case  ,		F"{split}.target"		)  ,		__snake_case		)
 return tmp_dir







class    lowerCamelCase__     ( lowerCAmelCase):





     @classmethod
     def   __A      (cls )  ->     Tuple:
      logging.disable(logging.CRITICAL )  # remove noisy download output from tracebacks
      return cls





     @slow
     @require_torch_gpu
     def   __A      (self )  ->     Dict:
      _lowercase							   =list_models()
      _lowercase							   ='''sshleifer'''
      _lowercase							   =[x.modelId for x in model_list if x.modelId.startswith(UpperCAmelCase )]
      _lowercase							   =['''sshleifer/blenderbot-3B''', '''sshleifer/blenderbot-90M''']
      _lowercase							   =[]
      for m in model_ids:
       if m in allowed_to_be_broken:
        continue
       try:
        AutoConfig.from_pretrained(UpperCAmelCase )
       except Exception:
        failures.append(UpperCAmelCase )
      assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"





     def   __A      (self )  ->     str:
      _lowercase							   ={'''student_encoder_layers''': 2, '''student_decoder_layers''': 1, '''no_teacher''': True}
      self._test_distiller_cli(UpperCAmelCase )





     def   __A      (self )  ->     Optional[Any]:
      _lowercase							   ={
          '''student_encoder_layers''': 2,
          '''student_decoder_layers''': 1,
          '''max_epochs''': 4,
          '''val_check_interval''': 0.25,
          '''alpha_hid''': 2.0,
          '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
      }
      _lowercase							   =self._test_distiller_cli(UpperCAmelCase	,  check_contents=UpperCAmelCase )

      _lowercase							   =list(Path(model.output_dir ).glob('''*.ckpt''' ) )
      self.assertEqual(1	,  len(UpperCAmelCase ) )
      _lowercase							   =list(Path(model.output_dir ).glob('''**/*.bin''' ) )
      self.assertEqual(len(UpperCAmelCase )	,  2 )
      _lowercase							   =lmap(str.strip	,  Path(model.hparams.data_dir ).joinpath('''test.source''' ).open().readlines() )
      _lowercase							   =tempfile.mktemp()  # XXX: not being cleaned up
      generate_summaries_or_translations(UpperCAmelCase	,  UpperCAmelCase	,  str(model.output_dir / '''best_tfmr''' ) )
      self.assertTrue(Path(UpperCAmelCase ).exists() )

      _lowercase							   =self.get_auto_remove_tmp_dir()
      convert_pl_to_hf(ckpts[0]	,  transformer_ckpts[0].parent	,  UpperCAmelCase )
      assert os.path.exists(os.path.join(UpperCAmelCase	,  '''pytorch_model.bin''' ) )





     def   __A      (self )  ->     List[str]:
      _lowercase							   =AutoModelForSeqaSeqLM.from_pretrained(UpperCAmelCase )
      _lowercase   ,					_lowercase							   =model.dummy_inputs['''input_ids'''], model.dummy_inputs['''attention_mask''']
      _lowercase							   =torch.tensor([[0, 4, 8, 2], [0, 8, 2, 1]]	,  dtype=torch.long	,  device=model.device )
      _lowercase							   =target_ids[:, :-1].contiguous()  # Why this line?
      _lowercase							   =target_ids[:, 1:].clone()  # why clone?
      _lowercase							   =model(
          UpperCAmelCase	,  attention_mask=UpperCAmelCase	,  decoder_input_ids=UpperCAmelCase	,  labels=UpperCAmelCase	,  use_cache=UpperCAmelCase ).loss

      _lowercase							   =model(UpperCAmelCase	,  attention_mask=UpperCAmelCase	,  decoder_input_ids=UpperCAmelCase	,  use_cache=UpperCAmelCase ).logits

      _lowercase							   =nn.functional.log_softmax(UpperCAmelCase	,  dim=-1 )
      _lowercase   ,					_lowercase							   =label_smoothed_nll_loss(
          UpperCAmelCase	,  UpperCAmelCase	,  0.1	,  ignore_index=model.config.pad_token_id )
      with self.assertRaises(UpperCAmelCase ):
       # TODO: understand why this breaks
       self.assertEqual(UpperCAmelCase	,  UpperCAmelCase )





     def   __A      (self )  ->     Any:
      _lowercase							   ={
          '''student_encoder_layers''': 2,
          '''student_decoder_layers''': 1,
          '''num_train_epochs''': 4,
          '''val_check_interval''': 0.25,
          '''alpha_hid''': 2.0,
          '''task''': '''translation''',
          '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
          '''tokenizer_name''': MBART_TINY,
          '''teacher''': MBART_TINY,
          '''src_lang''': '''en_XX''',
          '''tgt_lang''': '''ro_RO''',
      }
      _lowercase							   =self._test_distiller_cli(UpperCAmelCase	,  check_contents=UpperCAmelCase )
      assert model.model.config.model_type == "mbart"

      _lowercase							   =list(Path(model.output_dir ).glob('''*.ckpt''' ) )
      self.assertEqual(1	,  len(UpperCAmelCase ) )
      _lowercase							   =list(Path(model.output_dir ).glob('''**/*.bin''' ) )
      _lowercase							   =list(Path(model.output_dir ).glob('''best_tfmr/*''' ) )
      assert len(UpperCAmelCase ) > 2
      self.assertEqual(len(UpperCAmelCase )	,  2 )





     def   __A      (self )  ->     Any:
      _lowercase							   ={
          '''student_encoder_layers''': 1,
          '''student_decoder_layers''': 1,
          '''alpha_hid''': 2.0,
          '''teacher''': T5_TINY,
          '''model_name_or_path''': T5_TINY,
          '''tokenizer_name''': T5_TINY,
      }
      self._test_distiller_cli(UpperCAmelCase )





     def   __A      (self )  ->     Tuple:
      _lowercase							   ={
          '''teacher''': T5_TINY,
          '''student''': T5_TINIER,
          '''model_name_or_path''': T5_TINIER,
          '''tokenizer_name''': T5_TINIER,
      }
      self._test_distiller_cli(UpperCAmelCase )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase=True )  ->     Optional[int]:
      _lowercase							   ={
          '''label_smoothing''': 0.0,
          '''early_stopping_patience''': -1,
          '''train_batch_size''': 1,
          '''eval_batch_size''': 2,
          '''max_epochs''': 2,
          '''alpha_mlm''': 0.2,
          '''alpha_ce''': 0.8,
          '''do_predict''': True,
          '''model_name_or_path''': '''sshleifer/tinier_bart''',
          '''teacher''': CHEAP_ARGS['''model_name_or_path'''],
          '''val_check_interval''': 0.5,
      }
      default_updates.update(UpperCAmelCase )
      _lowercase							   =CHEAP_ARGS.copy()
      _lowercase							   =make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir() )
      _lowercase							   =self.get_auto_remove_tmp_dir()

      args_d.update(data_dir=UpperCAmelCase	,  output_dir=UpperCAmelCase	,  **UpperCAmelCase )
      _lowercase							   =distill_main(argparse.Namespace(**UpperCAmelCase ) )
      if not check_contents:
       return model
      _lowercase							   =os.listdir(UpperCAmelCase )
      _lowercase							   ={os.path.basename(UpperCAmelCase ) for p in contents}
      _lowercase							   =[p for p in contents if p.endswith('''ckpt''' )]
      assert len(UpperCAmelCase ) > 0

      self.assertIn('''test_generations.txt'''	,  UpperCAmelCase )
      self.assertIn('''test_results.txt'''	,  UpperCAmelCase )

      _lowercase							   =load_json(model.metrics_save_path )
      _lowercase							   =metrics['''val'''][-1]
      self.assertGreaterEqual(last_step_stats['''val_avg_gen_time''']	,  0.01 )
      self.assertGreaterEqual(1.0	,  last_step_stats['''val_avg_gen_time'''] )
      self.assertIsInstance(last_step_stats[f"val_avg_{model.val_metric}"]	,  UpperCAmelCase )
      _lowercase							   =int(args_d['''max_epochs'''] * (1 / args_d['''val_check_interval''']) + 1 )
      self.assertEqual(len(metrics['''val'''] )	,  UpperCAmelCase )
      self.assertEqual(len(metrics['''test'''] )	,  1 )
      return model







class    lowerCamelCase__     ( lowerCAmelCase):





     @parameterized.expand(
         [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY]	,  )
     def   __A      (self	,  UpperCAmelCase )  ->     Any:
      _lowercase							   =CHEAP_ARGS.copy()
      _lowercase							   ='''translation''' if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else '''summarization'''
      _lowercase							   =0.1 if task == '''translation''' else 0

      _lowercase							   =make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir() )
      _lowercase							   =self.get_auto_remove_tmp_dir()
      args_d.update(
          data_dir=UpperCAmelCase	,  model_name_or_path=UpperCAmelCase	,  tokenizer_name=UpperCAmelCase	,  train_batch_size=2	,  eval_batch_size=2	,  output_dir=UpperCAmelCase	,  do_predict=UpperCAmelCase	,  task=UpperCAmelCase	,  src_lang='''en_XX'''	,  tgt_lang='''ro_RO'''	,  freeze_encoder=UpperCAmelCase	,  freeze_embeds=UpperCAmelCase	,  )
      assert "n_train" in args_d
      _lowercase							   =argparse.Namespace(**UpperCAmelCase )
      _lowercase							   =main(UpperCAmelCase )

      _lowercase							   =module.model.get_input_embeddings()
      assert not input_embeds.weight.requires_grad
      if model == T5_TINY:
       _lowercase							   =module.model.lm_head
       assert not lm_head.weight.requires_grad
       assert (lm_head.weight == input_embeds.weight).all().item()
      elif model == FSMT_TINY:
       _lowercase							   =module.model.model
       _lowercase							   =fsmt.decoder.embed_positions
       assert not embed_pos.weight.requires_grad
       assert not fsmt.decoder.embed_tokens.weight.requires_grad
       # check that embeds are not the same
       assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
      else:
       _lowercase							   =module.model.model
       _lowercase							   =bart.decoder.embed_positions
       assert not embed_pos.weight.requires_grad
       assert not bart.shared.weight.requires_grad
       # check that embeds are the same
       assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
       assert bart.decoder.embed_tokens == bart.shared

      _lowercase							   =load_json(module.output_dir / '''text_batch.json''' )
      assert isinstance(UpperCAmelCase	,  UpperCAmelCase )
      assert len(UpperCAmelCase ) >= 4





     def   __A      (self )  ->     int:
      _lowercase							   =CHEAP_ARGS.copy()

      _lowercase							   ='''summarization'''
      _lowercase							   =make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir() )

      args_d.update(
          data_dir=UpperCAmelCase	,  tokenizer_name=UpperCAmelCase	,  train_batch_size=2	,  eval_batch_size=2	,  do_predict=UpperCAmelCase	,  task=UpperCAmelCase	,  src_lang='''en_XX'''	,  tgt_lang='''ro_RO'''	,  freeze_encoder=UpperCAmelCase	,  freeze_embeds=UpperCAmelCase	,  )

      # test models whose config includes the extra_model_args
      _lowercase							   =BART_TINY
      _lowercase							   =self.get_auto_remove_tmp_dir()
      _lowercase							   =args_d.copy()
      args_da.update(
          model_name_or_path=UpperCAmelCase	,  output_dir=UpperCAmelCase	,  )
      _lowercase							   =('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
      for p in extra_model_params:
       _lowercase							   =0.5
      _lowercase							   =argparse.Namespace(**UpperCAmelCase )
      _lowercase							   =main(UpperCAmelCase )
      for p in extra_model_params:
       assert getattr(model.config	,  UpperCAmelCase ) == 0.5, f"failed to override the model config for param {p}"

      # test models whose config doesn't include the extra_model_args
      _lowercase							   =T5_TINY
      _lowercase							   =self.get_auto_remove_tmp_dir()
      _lowercase							   =args_d.copy()
      args_da.update(
          model_name_or_path=UpperCAmelCase	,  output_dir=UpperCAmelCase	,  )
      _lowercase							   ='''encoder_layerdrop'''
      _lowercase							   =0.5
      _lowercase							   =argparse.Namespace(**UpperCAmelCase )
      with pytest.raises(UpperCAmelCase ) as excinfo:
       _lowercase							   =main(UpperCAmelCase )
      assert str(excinfo.value ) == f"model config doesn't have a `{unsupported_param}` attribute"





     def   __A      (self )  ->     str:
      _lowercase							   =CHEAP_ARGS.copy()

      _lowercase							   ='''summarization'''
      _lowercase							   =make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir() )

      _lowercase							   =BART_TINY
      _lowercase							   =self.get_auto_remove_tmp_dir()

      args_d.update(
          data_dir=UpperCAmelCase	,  model_name_or_path=UpperCAmelCase	,  output_dir=UpperCAmelCase	,  tokenizer_name=UpperCAmelCase	,  train_batch_size=2	,  eval_batch_size=2	,  do_predict=UpperCAmelCase	,  task=UpperCAmelCase	,  src_lang='''en_XX'''	,  tgt_lang='''ro_RO'''	,  freeze_encoder=UpperCAmelCase	,  freeze_embeds=UpperCAmelCase	,  )

      # emulate finetune.py
      _lowercase							   =argparse.ArgumentParser()
      _lowercase							   =pl.Trainer.add_argparse_args(UpperCAmelCase )
      _lowercase							   =SummarizationModule.add_model_specific_args(UpperCAmelCase	,  os.getcwd() )
      _lowercase							   ={'''--help''': True}

      # --help test
      with pytest.raises(UpperCAmelCase ) as excinfo:
       with CaptureStdout() as cs:
        _lowercase							   =parser.parse_args(UpperCAmelCase )
       assert False, "--help is expected to sys.exit"
      assert excinfo.type == SystemExit
      _lowercase							   =lightning_base.arg_to_scheduler_metavar
      assert expected in cs.out, "--help is expected to list the supported schedulers"

      # --lr_scheduler=non_existing_scheduler test
      _lowercase							   ='''non_existing_scheduler'''
      _lowercase							   ={f"--lr_scheduler={unsupported_param}"}
      with pytest.raises(UpperCAmelCase ) as excinfo:
       with CaptureStderr() as cs:
        _lowercase							   =parser.parse_args(UpperCAmelCase )
       assert False, "invalid argument is expected to sys.exit"
      assert excinfo.type == SystemExit
      _lowercase							   =f"invalid choice: '{unsupported_param}'"
      assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"

      # --lr_scheduler=existing_scheduler test
      _lowercase							   ='''cosine'''
      _lowercase							   =args_d.copy()
      _lowercase							   =supported_param
      _lowercase							   =argparse.Namespace(**UpperCAmelCase )
      _lowercase							   =main(UpperCAmelCase )
      assert (
          getattr(model.hparams	,  '''lr_scheduler''' ) == supported_param
      ), f"lr_scheduler={supported_param} shouldn't fail"





import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
  import jax
  import jax.numpy as jnp

  from transformers.modeling_flax_pytorch_utils import (
      convert_pytorch_state_dict_to_flax,
      load_flax_weights_in_pytorch_model,
  )
  from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
  import torch







class    lowerCamelCase__     :





     def __init__(self	,  UpperCAmelCase	,  UpperCAmelCase=1_4	,  UpperCAmelCase=7	,  UpperCAmelCase=True	,  UpperCAmelCase=True	,  UpperCAmelCase=False	,  UpperCAmelCase=True	,  UpperCAmelCase=9_9	,  UpperCAmelCase=3_2	,  UpperCAmelCase=5	,  UpperCAmelCase=4	,  UpperCAmelCase=3_7	,  UpperCAmelCase="gelu"	,  UpperCAmelCase=0.1	,  UpperCAmelCase=0.1	,  UpperCAmelCase=5_1_2	,  UpperCAmelCase=0.02	,  )  ->     Any:
      _lowercase							   =parent
      _lowercase							   =batch_size
      _lowercase							   =seq_length
      _lowercase							   =is_training
      _lowercase							   =use_input_mask
      _lowercase							   =use_token_type_ids
      _lowercase							   =use_labels
      _lowercase							   =vocab_size
      _lowercase							   =hidden_size
      _lowercase							   =num_hidden_layers
      _lowercase							   =num_attention_heads
      _lowercase							   =intermediate_size
      _lowercase							   =hidden_act
      _lowercase							   =hidden_dropout_prob
      _lowercase							   =attention_probs_dropout_prob
      _lowercase							   =max_position_embeddings
      _lowercase							   =initializer_range
      _lowercase							   =None
      _lowercase							   =vocab_size - 1
      _lowercase							   =vocab_size - 1
      _lowercase							   =vocab_size - 1





     def   __A      (self )  ->     int:
      _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  self.vocab_size )

      _lowercase							   =None
      if self.use_input_mask:
       _lowercase							   =random_attention_mask([self.batch_size, self.seq_length] )

      _lowercase							   =GPTaConfig(
          vocab_size=self.vocab_size	,  n_embd=self.hidden_size	,  n_layer=self.num_hidden_layers	,  n_head=self.num_attention_heads	,  n_positions=self.max_position_embeddings	,  use_cache=UpperCAmelCase	,  bos_token_id=self.bos_token_id	,  eos_token_id=self.eos_token_id	,  pad_token_id=self.pad_token_id	,  )

      return (config, input_ids, input_mask)





     def   __A      (self )  ->     List[Any]:
      _lowercase							   =self.prepare_config_and_inputs()
      _lowercase   ,					_lowercase   ,					_lowercase							   =config_and_inputs
      _lowercase							   ={'''input_ids''': input_ids, '''attention_mask''': attention_mask}
      return config, inputs_dict





     def   __A      (self )  ->     Tuple:
      _lowercase   ,					_lowercase   ,					_lowercase							   =self.prepare_config_and_inputs()

      _lowercase							   =floats_tensor([self.batch_size, self.seq_length, self.hidden_size] )
      _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  vocab_size=2 )

      return (
          config,
          input_ids,
          attention_mask,
          encoder_hidden_states,
          encoder_attention_mask,
      )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase )  ->     Dict:
      _lowercase							   =2_0
      _lowercase							   =model_class_name(UpperCAmelCase )

      _lowercase							   =model.init_cache(input_ids.shape[0]	,  UpperCAmelCase )
      _lowercase							   =jnp.ones((input_ids.shape[0], max_decoder_length)	,  dtype='''i4''' )

      _lowercase							   =jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1 )[None, :]	,  (input_ids.shape[0], input_ids.shape[-1] - 1) )
      _lowercase							   =model(
          input_ids[:, :-1]	,  attention_mask=UpperCAmelCase	,  past_key_values=UpperCAmelCase	,  position_ids=UpperCAmelCase	,  )

      _lowercase							   =jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]	,  dtype='''i4''' )
      _lowercase							   =model(
          input_ids[:, -1:]	,  attention_mask=UpperCAmelCase	,  past_key_values=outputs_cache.past_key_values	,  position_ids=UpperCAmelCase	,  )

      _lowercase							   =model(UpperCAmelCase )

      _lowercase							   =np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
      self.parent.assertTrue(diff < 1e-3	,  msg=f"Max diff is {diff}" )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase )  ->     List[str]:
      _lowercase							   =2_0
      _lowercase							   =model_class_name(UpperCAmelCase )

      _lowercase							   =jnp.concatenate(
          [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1]) )]	,  axis=-1	,  )

      _lowercase							   =model.init_cache(input_ids.shape[0]	,  UpperCAmelCase )
      _lowercase							   =jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1 )[None, :]	,  (input_ids.shape[0], input_ids.shape[-1] - 1) )

      _lowercase							   =model(
          input_ids[:, :-1]	,  attention_mask=UpperCAmelCase	,  past_key_values=UpperCAmelCase	,  position_ids=UpperCAmelCase	,  )
      _lowercase							   =jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]	,  dtype='''i4''' )
      _lowercase							   =model(
          input_ids[:, -1:]	,  past_key_values=outputs_cache.past_key_values	,  attention_mask=UpperCAmelCase	,  position_ids=UpperCAmelCase	,  )

      _lowercase							   =model(UpperCAmelCase	,  attention_mask=UpperCAmelCase )

      _lowercase							   =np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
      self.parent.assertTrue(diff < 1e-3	,  msg=f"Max diff is {diff}" )







@require_flax
class    lowerCamelCase__     ( lowerCAmelCase     ,							lowerCAmelCase     ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__       =    (FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
     SCREAMING_SNAKE_CASE__       =    (FlaxGPTaLMHeadModel,) if is_flax_available() else ()





     def   __A      (self )  ->     str:
      _lowercase							   =FlaxGPTaModelTester(self )





     def   __A      (self )  ->     Optional[int]:
      for model_class_name in self.all_model_classes:
       _lowercase   ,					_lowercase   ,					_lowercase							   =self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward(UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase )





     def   __A      (self )  ->     Dict:
      for model_class_name in self.all_model_classes:
       _lowercase   ,					_lowercase   ,					_lowercase							   =self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward_with_attn_mask(
           UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase )





     @slow
     def   __A      (self )  ->     Dict:
      _lowercase							   =GPTaTokenizer.from_pretrained('''gpt2'''	,  pad_token='''</s>'''	,  padding_side='''left''' )
      _lowercase							   =tokenizer(['''Hello this is a long string''', '''Hey''']	,  return_tensors='''np'''	,  padding=UpperCAmelCase	,  truncation=UpperCAmelCase )

      _lowercase							   =FlaxGPTaLMHeadModel.from_pretrained('''gpt2''' )
      _lowercase							   =False
      _lowercase							   =model.config.eos_token_id

      _lowercase							   =jax.jit(model.generate )

      _lowercase							   =jit_generate(inputs['''input_ids''']	,  attention_mask=inputs['''attention_mask'''] ).sequences

      _lowercase							   =tokenizer.batch_decode(UpperCAmelCase	,  skip_special_tokens=UpperCAmelCase )

      _lowercase							   =[
          '''Hello this is a long string of words. I\'m going to start with the first one.\n''',
          '''Hey, I\'m not sure if I\'m going to be able to do''',
      ]

      self.assertListEqual(UpperCAmelCase	,  UpperCAmelCase )





     @is_pt_flax_cross_test
     def   __A      (self )  ->     Optional[Any]:
      _lowercase   ,					_lowercase							   =self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__ ):
        # prepare inputs
        _lowercase							   =self._prepare_for_class(UpperCAmelCase	,  UpperCAmelCase )
        _lowercase							   ={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

        # load corresponding PyTorch class
        _lowercase							   =model_class.__name__[4:]  # Skip the "Flax" at the beginning
        _lowercase							   =getattr(UpperCAmelCase	,  UpperCAmelCase )

        _lowercase   ,					_lowercase							   =pt_inputs['''input_ids'''].shape
        _lowercase							   =np.random.randint(0	,  seq_length - 1	,  size=(batch_size,) )
        for batch_idx, start_index in enumerate(UpperCAmelCase ):
         _lowercase							   =0
         _lowercase							   =1
         _lowercase							   =0
         _lowercase							   =1
        _lowercase							   =pt_model_class(UpperCAmelCase ).eval()
        _lowercase							   =model_class(UpperCAmelCase	,  dtype=jnp.floataa )

        _lowercase							   =convert_pytorch_state_dict_to_flax(pt_model.state_dict()	,  UpperCAmelCase )
        _lowercase							   =fx_state

        with torch.no_grad():
         _lowercase							   =pt_model(**UpperCAmelCase ).to_tuple()

        _lowercase							   =fx_model(**UpperCAmelCase ).to_tuple()
        self.assertEqual(len(UpperCAmelCase )	,  len(UpperCAmelCase )	,  '''Output lengths differ between Flax and PyTorch''' )
        for fx_output, pt_output in zip(UpperCAmelCase	,  UpperCAmelCase ):
         self.assert_almost_equals(fx_output[:, -1]	,  pt_output[:, -1].numpy()	,  4e-2 )

        with tempfile.TemporaryDirectory() as tmpdirname:
         pt_model.save_pretrained(UpperCAmelCase )
         _lowercase							   =model_class.from_pretrained(UpperCAmelCase	,  from_pt=UpperCAmelCase )

        _lowercase							   =fx_model_loaded(**UpperCAmelCase ).to_tuple()
        self.assertEqual(
            len(UpperCAmelCase )	,  len(UpperCAmelCase )	,  '''Output lengths differ between Flax and PyTorch''' )
        for fx_output_loaded, pt_output in zip(UpperCAmelCase	,  UpperCAmelCase ):
         self.assert_almost_equals(fx_output_loaded[:, -1]	,  pt_output[:, -1].numpy()	,  4e-2 )





     @is_pt_flax_cross_test
     def   __A      (self )  ->     Optional[Any]:
      _lowercase   ,					_lowercase							   =self.model_tester.prepare_config_and_inputs_for_common()
      for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__ ):
        # prepare inputs
        _lowercase							   =self._prepare_for_class(UpperCAmelCase	,  UpperCAmelCase )
        _lowercase							   ={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

        # load corresponding PyTorch class
        _lowercase							   =model_class.__name__[4:]  # Skip the "Flax" at the beginning
        _lowercase							   =getattr(UpperCAmelCase	,  UpperCAmelCase )

        _lowercase							   =pt_model_class(UpperCAmelCase ).eval()
        _lowercase							   =model_class(UpperCAmelCase	,  dtype=jnp.floataa )

        _lowercase							   =load_flax_weights_in_pytorch_model(UpperCAmelCase	,  fx_model.params )
        _lowercase   ,					_lowercase							   =pt_inputs['''input_ids'''].shape
        _lowercase							   =np.random.randint(0	,  seq_length - 1	,  size=(batch_size,) )
        for batch_idx, start_index in enumerate(UpperCAmelCase ):
         _lowercase							   =0
         _lowercase							   =1
         _lowercase							   =0
         _lowercase							   =1

        # make sure weights are tied in PyTorch
        pt_model.tie_weights()

        with torch.no_grad():
         _lowercase							   =pt_model(**UpperCAmelCase ).to_tuple()

        _lowercase							   =fx_model(**UpperCAmelCase ).to_tuple()
        self.assertEqual(len(UpperCAmelCase )	,  len(UpperCAmelCase )	,  '''Output lengths differ between Flax and PyTorch''' )
        for fx_output, pt_output in zip(UpperCAmelCase	,  UpperCAmelCase ):
         self.assert_almost_equals(fx_output[:, -1]	,  pt_output[:, -1].numpy()	,  4e-2 )

        with tempfile.TemporaryDirectory() as tmpdirname:
         fx_model.save_pretrained(UpperCAmelCase )
         _lowercase							   =pt_model_class.from_pretrained(UpperCAmelCase	,  from_flax=UpperCAmelCase )

        with torch.no_grad():
         _lowercase							   =pt_model_loaded(**UpperCAmelCase ).to_tuple()

        self.assertEqual(
            len(UpperCAmelCase )	,  len(UpperCAmelCase )	,  '''Output lengths differ between Flax and PyTorch''' )
        for fx_output, pt_output in zip(UpperCAmelCase	,  UpperCAmelCase ):
         self.assert_almost_equals(fx_output[:, -1]	,  pt_output[:, -1].numpy()	,  4e-2 )





     @slow
     def   __A      (self )  ->     Union[str, Any]:
      for model_class_name in self.all_model_classes:
       _lowercase							   =model_class_name.from_pretrained('''gpt2'''	,  from_pt=UpperCAmelCase )
       _lowercase							   =model(np.ones((1, 1) ) )
       self.assertIsNotNone(UpperCAmelCase )

