
import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      FunnelBaseModel,
      FunnelForMaskedLM,
      FunnelForMultipleChoice,
      FunnelForPreTraining,
      FunnelForQuestionAnswering,
      FunnelForSequenceClassification,
      FunnelForTokenClassification,
      FunnelModel,
  )

class __snake_case	:
    def __init__(						self	,snake_case	,snake_case=13	,snake_case=7	,snake_case=True	,snake_case=True	,snake_case=True	,snake_case=True	,snake_case=99	,snake_case=[1, 1, 2]	,snake_case=1	,snake_case=32	,snake_case=4	,snake_case=8	,snake_case=37	,snake_case="gelu_new"	,snake_case=0.1	,snake_case=0.1	,snake_case=0.0	,snake_case=512	,snake_case=3	,snake_case=0.02	,snake_case=3	,snake_case=4	,snake_case=None	,snake_case=False	,):



           '''simple docstring'''







           lowercase							:  int 			=   parent
           lowercase							:  List[str] 			=   batch_size
           lowercase							:  Tuple 			=   seq_length
           lowercase							:  Any 			=   is_training
           lowercase							:  Any 			=   use_input_mask
           lowercase							:  str 			=   use_token_type_ids
           lowercase							:  str 			=   use_labels
           lowercase							:  int 			=   vocab_size
           lowercase							:  Dict 			=   block_sizes
           lowercase							:  Union[str, Any] 			=   num_decoder_layers
           lowercase							:  Tuple 			=   d_model
           lowercase							:  Dict 			=   n_head
           lowercase							:  Dict 			=   d_head
           lowercase							:  Optional[int] 			=   d_inner
           lowercase							:  List[str] 			=   hidden_act
           lowercase							:  Dict 			=   hidden_dropout
           lowercase							:  Union[str, Any] 			=   attention_dropout
           lowercase							:  Any 			=   activation_dropout
           lowercase							:  Tuple 			=   max_position_embeddings
           lowercase							:  Any 			=   type_vocab_size
           lowercase							:  str 			=   2
           lowercase							:  Optional[int] 			=   num_labels
           lowercase							:  int 			=   num_choices
           lowercase							:  Union[str, Any] 			=   scope
           lowercase							:  int 			=   initializer_std

           # Used in the tests to check the size of the first attention layer
           lowercase							:  Optional[Any] 			=   n_head
           # Used in the tests to check the size of the first hidden state
           lowercase							:  List[Any] 			=   self.d_model
           # Used in the tests to check the number of output hidden states/attentions
           lowercase							:  List[str] 			=   sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
           # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
           # the last hidden state of the first block (which is the first hidden state of the decoder).
           if not base:
                  lowercase							:  Any 			=   self.num_hidden_layers + 2
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Optional[Any] 			=   ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size   )

           lowercase							:  Dict 			=   None
           if self.use_input_mask:
                  lowercase							:  Dict 			=   ids_tensor([self.batch_size, self.seq_length]	,vocab_size=2   )

           lowercase							:  List[Any] 			=   None
           if self.use_token_type_ids:
                  lowercase							:  Any 			=   ids_tensor([self.batch_size, self.seq_length]	,self.type_vocab_size   )

           lowercase							:  List[Any] 			=   None
           lowercase							:  List[Any] 			=   None
           lowercase							:  Union[str, Any] 			=   None
           if self.use_labels:
                  lowercase							:  List[str] 			=   ids_tensor([self.batch_size]	,self.type_sequence_label_size   )
                  lowercase							:  Tuple 			=   ids_tensor([self.batch_size, self.seq_length]	,self.num_labels   )
                  lowercase							:  int 			=   ids_tensor([self.batch_size]	,self.num_choices   )
                  lowercase							:  List[Any] 			=   ids_tensor([self.batch_size, self.seq_length]	,1   )

           lowercase							:  Tuple 			=   self.get_config()

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               fake_token_labels,
           )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           return FunnelConfig(
               vocab_size=self.vocab_size	,block_sizes=self.block_sizes	,num_decoder_layers=self.num_decoder_layers	,d_model=self.d_model	,n_head=self.n_head	,d_head=self.d_head	,d_inner=self.d_inner	,hidden_act=self.hidden_act	,hidden_dropout=self.hidden_dropout	,attention_dropout=self.attention_dropout	,activation_dropout=self.activation_dropout	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,initializer_std=self.initializer_std	,)
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  Optional[Any] 			=   FunnelModel(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Tuple 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case   )
           lowercase							:  List[Any] 			=   model(snake_case	,token_type_ids=snake_case   )
           lowercase							:  Union[str, Any] 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.d_model)   )

           lowercase							:  Dict 			=   False
           lowercase							:  Any 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.d_model)   )

           lowercase							:  Union[str, Any] 			=   False
           lowercase							:  Optional[Any] 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.d_model)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  str 			=   FunnelBaseModel(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Tuple 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case   )
           lowercase							:  Dict 			=   model(snake_case	,token_type_ids=snake_case   )
           lowercase							:  str 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, 2, self.d_model)   )

           lowercase							:  List[str] 			=   False
           lowercase							:  List[str] 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, 3, self.d_model)   )

           lowercase							:  int 			=   False
           lowercase							:  List[Any] 			=   model(snake_case   )
           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, 2, self.d_model)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  Dict 			=   self.num_labels
           lowercase							:  str 			=   FunnelForPreTraining(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Dict 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  Dict 			=   FunnelForMaskedLM(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Union[str, Any] 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.num_labels
           lowercase							:  List[str] 			=   FunnelForSequenceClassification(snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Optional[int] 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.num_labels)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  List[str] 			=   self.num_choices
           lowercase							:  Tuple 			=   FunnelForMultipleChoice(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Dict 			=   input_ids.unsqueeze(1   ).expand(-1	,self.num_choices	,-1   ).contiguous()
           lowercase							:  List[Any] 			=   token_type_ids.unsqueeze(1   ).expand(-1	,self.num_choices	,-1   ).contiguous()
           lowercase							:  List[str] 			=   input_mask.unsqueeze(1   ).expand(-1	,self.num_choices	,-1   ).contiguous()
           lowercase							:  Any 			=   model(
               snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,labels=snake_case	,)
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.num_choices)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.num_labels
           lowercase							:  Any 			=   FunnelForTokenClassification(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Tuple 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.num_labels)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,):



           '''simple docstring'''







           lowercase							:  Optional[int] 			=   FunnelForQuestionAnswering(config=snake_case   )
           model.to(snake_case   )
           model.eval()
           lowercase							:  Dict 			=   model(
               snake_case	,attention_mask=snake_case	,token_type_ids=snake_case	,start_positions=snake_case	,end_positions=snake_case	,)
           self.parent.assertEqual(result.start_logits.shape	,(self.batch_size, self.seq_length)   )
           self.parent.assertEqual(result.end_logits.shape	,(self.batch_size, self.seq_length)   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Tuple 			=   self.prepare_config_and_inputs()
           (
               (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    
           )							:  Tuple 			=   config_and_inputs
           lowercase							:  Optional[int] 			=   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
           return config, inputs_dict






@require_torch
class __snake_case	(   lowerCAmelCase      ,			lowerCAmelCase      ,			unittest.TestCase  ):
    _a	:		List[Any]=       False
    _a	:		Dict=       False
    _a	:		Optional[Any]=       (
        (
            FunnelModel,
            FunnelForMaskedLM,
            FunnelForPreTraining,
            FunnelForQuestionAnswering,
            FunnelForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    _a	:		Optional[Any]=       (
        {
            "feature-extraction": (FunnelBaseModel, FunnelModel),
            "fill-mask": FunnelForMaskedLM,
            "question-answering": FunnelForQuestionAnswering,
            "text-classification": FunnelForSequenceClassification,
            "token-classification": FunnelForTokenClassification,
            "zero-shot": FunnelForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case=False   ):



           '''simple docstring'''







           lowercase							:  Union[str, Any] 			=   super()._prepare_for_class(snake_case	,snake_case	,return_labels=snake_case   )

           if return_labels:
                  if model_class in get_values(snake_case   ):
                         lowercase							:  List[Any] 			=   torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length)	,dtype=torch.long	,device=snake_case   )
           return inputs_dict
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Optional[int] 			=   FunnelModelTester(self   )
           lowercase							:  Optional[int] 			=   ConfigTester(self	,config_class=snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           self.config_tester.run_common_tests()
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[str] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Optional[int] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Union[str, Any] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case   ):



           '''simple docstring'''







           if hasattr(snake_case	,"""weight"""   ) and module.weight is not None:
                  module.weight.data.fill_(3   )
           if hasattr(snake_case	,"""bias"""   ) and module.bias is not None:
                  module.bias.data.fill_(3   )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(snake_case	,snake_case   ) and getattr(snake_case	,snake_case   ) is not None:
                         lowercase							:  int 			=   getattr(snake_case	,snake_case   )
                         weight.data.fill_(3   )






@require_torch
class __snake_case	(   lowerCAmelCase      ,			unittest.TestCase  ):
    _a	:		int=       False
    _a	:		List[Any]=       False
    _a	:		Union[str, Any]=       (
        (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
    )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  int 			=   FunnelModelTester(self	,base=snake_case   )
           lowercase							:  int 			=   ConfigTester(self	,config_class=snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           self.config_tester.run_common_tests()
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Dict 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_base_model(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[str] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase			,    lowercase							:  List[str] 			=   self.model_tester.prepare_config_and_inputs_for_common()
           lowercase							:  Optional[int] 			=   True

           for model_class in self.all_model_classes:
                  if model_class.__name__ == "FunnelBaseModel":
                         continue
                  lowercase							:  List[str] 			=   model_class(snake_case   )
                  model.to(snake_case   )
                  model.train()
                  lowercase							:  Optional[Any] 			=   self._prepare_for_class(snake_case	,snake_case	,return_labels=snake_case   )
                  lowercase							:  str 			=   model(**snake_case   ).loss
                  loss.backward()
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case   ):



           '''simple docstring'''







           if hasattr(snake_case	,"""weight"""   ) and module.weight is not None:
                  module.weight.data.fill_(3   )
           if hasattr(snake_case	,"""bias"""   ) and module.bias is not None:
                  module.bias.data.fill_(3   )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(snake_case	,snake_case   ) and getattr(snake_case	,snake_case   ) is not None:
                         lowercase							:  Optional[Any] 			=   getattr(snake_case	,snake_case   )
                         weight.data.fill_(3   )






@require_torch
@require_sentencepiece
@require_tokenizers
class __snake_case	(   unittest.TestCase  ):
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Dict 			=   13
           lowercase							:  List[str] 			=   7
           lowercase							:  List[str] 			=   torch.arange(0	,batch_size * sequence_length   ).long().reshape(snake_case	,snake_case   )
           lowercase							:  Optional[int] 			=   [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
           lowercase							:  List[str] 			=   torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )

           lowercase							:  List[Any] 			=   FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""   )
           lowercase							:  Tuple 			=   model(snake_case	,token_type_ids=snake_case   )[0].abs()

           lowercase							:  Optional[int] 			=   torch.tensor(2_344.8_352   )
           lowercase							:  Any 			=   torch.tensor(0.8_052   )
           self.assertTrue(torch.allclose(output.sum()	,snake_case	,atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()	,snake_case	,atol=1e-4   )   )

           lowercase							:  Optional[Any] 			=   torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
           lowercase							:  List[Any] 			=   model(snake_case	,attention_mask=snake_case	,token_type_ids=snake_case   )[0].abs()

           lowercase							:  str 			=   torch.tensor(2_343.8_425   )
           lowercase							:  List[Any] 			=   torch.tensor(0.8_049   )
           self.assertTrue(torch.allclose(output.sum()	,snake_case	,atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()	,snake_case	,atol=1e-4   )   )
    @slow
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[str] 			=   FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""   )
           lowercase							:  Any 			=   FunnelModel.from_pretrained("""huggingface/funnel-small"""   )
           lowercase							:  List[str] 			=   tokenizer("""Hello! I am the Funnel Transformer model."""	,return_tensors="""pt"""   )
           lowercase							:  Union[str, Any] 			=   model(**snake_case   )[0]

           lowercase							:  List[Any] 			=   torch.tensor(235.7_246   )
           lowercase							:  Union[str, Any] 			=   torch.tensor(0.0_256   )
           self.assertTrue(torch.allclose(output.sum()	,snake_case	,atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()	,snake_case	,atol=1e-4   )   )


import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
      RwkvForCausalLM,
      RwkvModel,
  )
  from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
  lowercase						:      Any							=							False

class __snake_case	:
    def __init__(						self	,snake_case	,snake_case=14	,snake_case=7	,snake_case=True	,snake_case=False	,snake_case=True	,snake_case=True	,snake_case=True	,snake_case=99	,snake_case=32	,snake_case=5	,snake_case=37	,snake_case="gelu"	,snake_case=0.1	,snake_case=0.1	,snake_case=512	,snake_case=16	,snake_case=2	,snake_case=3	,snake_case=4	,snake_case=None	,):



           '''simple docstring'''







           lowercase							:  Optional[Any] 			=   parent
           lowercase							:  str 			=   batch_size
           lowercase							:  str 			=   seq_length
           lowercase							:  int 			=   is_training
           lowercase							:  Optional[Any] 			=   use_token_type_ids
           lowercase							:  List[str] 			=   use_input_mask
           lowercase							:  List[Any] 			=   use_labels
           lowercase							:  List[Any] 			=   use_mc_token_ids
           lowercase							:  int 			=   vocab_size
           lowercase							:  Any 			=   hidden_size
           lowercase							:  Optional[int] 			=   num_hidden_layers
           lowercase							:  Optional[int] 			=   intermediate_size
           lowercase							:  Optional[int] 			=   hidden_act
           lowercase							:  Optional[int] 			=   hidden_dropout_prob
           lowercase							:  List[str] 			=   attention_probs_dropout_prob
           lowercase							:  Dict 			=   max_position_embeddings
           lowercase							:  Optional[Any] 			=   type_vocab_size
           lowercase							:  str 			=   type_sequence_label_size
           lowercase							:  int 			=   num_labels
           lowercase							:  List[Any] 			=   num_choices
           lowercase							:  int 			=   scope
           lowercase							:  Any 			=   vocab_size - 1
           lowercase							:  str 			=   vocab_size - 1
           lowercase							:  int 			=   vocab_size - 1
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           return RwkvConfig.from_pretrained("""sgugger/rwkv-4-pile-7b"""   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case=False	,snake_case=False	,snake_case=False   ):



           '''simple docstring'''







           lowercase							:  Optional[Any] 			=   ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size   )

           lowercase							:  Tuple 			=   None
           if self.use_input_mask:
                  lowercase							:  List[str] 			=   random_attention_mask([self.batch_size, self.seq_length]   )

           lowercase							:  List[Any] 			=   None
           if self.use_token_type_ids:
                  lowercase							:  str 			=   ids_tensor([self.batch_size, self.seq_length]	,self.type_vocab_size   )

           lowercase							:  Any 			=   None
           if self.use_mc_token_ids:
                  lowercase							:  Tuple 			=   ids_tensor([self.batch_size, self.num_choices]	,self.seq_length   )

           lowercase							:  Union[str, Any] 			=   None
           lowercase							:  List[Any] 			=   None
           lowercase							:  Tuple 			=   None
           if self.use_labels:
                  lowercase							:  int 			=   ids_tensor([self.batch_size]	,self.type_sequence_label_size   )
                  lowercase							:  str 			=   ids_tensor([self.batch_size, self.seq_length]	,self.num_labels   )
                  lowercase							:  Tuple 			=   ids_tensor([self.batch_size]	,self.num_choices   )

           lowercase							:  List[str] 			=   self.get_config(
               gradient_checkpointing=snake_case	,scale_attn_by_inverse_layer_idx=snake_case	,reorder_and_upcast_attn=snake_case	,)

           return (
               config,
               input_ids,
               input_mask,
               None,
               token_type_ids,
               mc_token_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case=False	,snake_case=False	,snake_case=False   ):



           '''simple docstring'''







           return RwkvConfig(
               vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,intermediate_size=self.intermediate_size	,activation_function=self.hidden_act	,resid_pdrop=self.hidden_dropout_prob	,attn_pdrop=self.attention_probs_dropout_prob	,n_positions=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,use_cache=snake_case	,bos_token_id=self.bos_token_id	,eos_token_id=self.eos_token_id	,pad_token_id=self.pad_token_id	,gradient_checkpointing=snake_case	,scale_attn_by_inverse_layer_idx=snake_case	,reorder_and_upcast_attn=snake_case	,)
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.get_config()
           lowercase							:  str 			=   300
           return config
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           (
               (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    
           )							:  Dict 			=   self.prepare_config_and_inputs()

           lowercase							:  List[str] 			=   floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
           lowercase							:  Tuple 			=   ids_tensor([self.batch_size, self.seq_length]	,vocab_size=2   )

           return (
               config,
               input_ids,
               input_mask,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,*snake_case   ):



           '''simple docstring'''







           lowercase							:  Optional[Any] 			=   True
           lowercase							:  Dict 			=   RwkvModel(config=snake_case   )
           model.to(snake_case   )
           model.eval()

           lowercase							:  Optional[Any] 			=   model(snake_case   )

           self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )
           self.parent.assertEqual(len(result.hidden_states   )	,config.num_hidden_layers + 1   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,*snake_case   ):



           '''simple docstring'''







           lowercase							:  int 			=   RwkvForCausalLM(snake_case   )
           model.to(snake_case   )
           model.eval()

           lowercase							:  Optional[int] 			=   model(snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.loss.shape	,()   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,*snake_case   ):



           '''simple docstring'''







           lowercase							:  str 			=   RwkvModel(config=snake_case   )
           model.to(snake_case   )
           model.eval()

           lowercase							:  int 			=   model(snake_case   )
           lowercase							:  int 			=   outputs.last_hidden_state

           lowercase							:  Dict 			=   model(input_ids[:, :2]   )
           lowercase							:  Optional[int] 			=   outputs.last_hidden_state

           # Using the state computed on the first inputs, we will get the same output
           lowercase							:  int 			=   model(input_ids[:, 2:]	,state=outputs.state   )
           lowercase							:  Union[str, Any] 			=   outputs.last_hidden_state

           self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]	,dim=1   )	,snake_case	,atol=1e-5   )   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case	,snake_case	,snake_case	,*snake_case	,snake_case=False   ):



           '''simple docstring'''







           lowercase							:  Dict 			=   RwkvForCausalLM(snake_case   )
           model.to(snake_case   )
           if gradient_checkpointing:
                  model.gradient_checkpointing_enable()

           lowercase							:  List[str] 			=   model(snake_case	,labels=snake_case   )
           self.parent.assertEqual(result.loss.shape	,()   )
           self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)   )
           result.loss.backward()
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   self.prepare_config_and_inputs()

           (
               (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    (
               lowercase
           )			,    
           )							:  Union[str, Any] 			=   config_and_inputs

           lowercase							:  Optional[Any] 			=   {"""input_ids""": input_ids}

           return config, inputs_dict






@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0      ,			reason="See https://github.com/huggingface/transformers/pull/24204"  )
@require_torch
class __snake_case	(   lowerCAmelCase      ,			lowerCAmelCase      ,			lowerCAmelCase      ,			unittest.TestCase  ):
    _a	:		str=       (RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
    _a	:		Tuple=       (
        {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {}
    )
    # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
    _a	:		List[str]=       False
    _a	:		int=       False
    _a	:		Optional[Any]=       False
    _a	:		List[Any]=       False
    _a	:		int=       False  # Rwkv does not support head masking
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Dict 			=   RwkvModelTester(self   )
           lowercase							:  Any 			=   ConfigTester(
               self	,config_class=snake_case	,n_embd=37	,common_properties=["""hidden_size""", """num_hidden_layers"""]   )
    def    _SCREAMING_SNAKE_CASE (						self	,snake_case	,snake_case	,snake_case=None   ):



           '''simple docstring'''







           if isinstance(snake_case	,torch.Tensor   ):
                  lowercase			,    lowercase							:  Tuple 			=   member.max().item(), member.min().item()
           elif isinstance(snake_case	,snake_case   ) or isinstance(snake_case	,snake_case   ):
                  lowercase			,    lowercase							:  Dict 			=   max(snake_case   ), min(snake_case   )

           if not isinstance(snake_case	,snake_case   ):
                  raise TypeError("""container should be a list or tuple"""   )
           elif len(snake_case   ) != 2:
                  raise ValueError("""container should have 2 elements"""   )

           lowercase			,    lowercase							:  int 			=   container

           lowercase							:  Union[str, Any] 			=   (min_value >= expected_min) and (max_value <= expected_max)

           if not is_inside_interval:
                  lowercase							:  Optional[int] 			=   """%s not found in %s""" % (safe_repr(snake_case   ), safe_repr(snake_case   ))
                  self.fail(self._formatMessage(snake_case	,snake_case   )   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           self.config_tester.run_common_tests()
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Any 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_rwkv_model(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Any 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_causl_lm(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Any 			=   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_state_equivalency(*snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase			,    lowercase							:  Union[str, Any] 			=   self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                  lowercase							:  List[Any] 			=   model_class(config=snake_case   )
                  for name, param in model.named_parameters():
                         if "time_decay" in name:
                                if param.requires_grad:
                                       self.assertTrue(param.data.max().item() == 3.0   )
                                       self.assertTrue(param.data.min().item() == -5.0   )
                         elif "time_first" in name:
                                if param.requires_grad:
                                       # check if it's a ones like
                                       self.assertTrue(torch.allclose(param.data	,torch.ones_like(param.data   )	,atol=1e-5	,rtol=1e-5   )   )
                         elif any(x in name for x in ["""time_mix_key""", """time_mix_receptance"""]   ):
                                if param.requires_grad:
                                       self.assertInterval(
                                           param.data	,[0.0, 1.0]	,msg=f"Parameter {name} of model {model_class} seems not properly initialized"	,)
                         elif "time_mix_value" in name:
                                if param.requires_grad:
                                       self.assertInterval(
                                           param.data	,[0.0, 1.3]	,msg=f"Parameter {name} of model {model_class} seems not properly initialized"	,)
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase			,    lowercase							:  str 			=   self.model_tester.prepare_config_and_inputs_for_common()
           lowercase							:  List[Any] 			=   True

           lowercase							:  int 			=   getattr(self.model_tester	,"""seq_length"""	,snake_case   )

           for model_class in self.all_model_classes:
                  lowercase							:  Union[str, Any] 			=   True
                  lowercase							:  str 			=   False
                  lowercase							:  int 			=   True
                  lowercase							:  List[str] 			=   model_class(snake_case   )
                  model.to(snake_case   )
                  model.eval()

                  lowercase							:  Tuple 			=   self._prepare_for_class(snake_case	,snake_case   )
                  lowercase							:  Optional[Any] 			=   inputs["""input_ids"""].shape[0]
                  with torch.no_grad():
                         lowercase							:  Any 			=   model(**snake_case   )
                  lowercase							:  List[Any] 			=   outputs.attentions
                  self.assertEqual(len(snake_case   )	,self.model_tester.num_hidden_layers   )

                  # check that output_attentions also work using config
                  del inputs_dict["output_attentions"]
                  lowercase							:  str 			=   True
                  lowercase							:  Any 			=   model_class(snake_case   )
                  model.to(snake_case   )
                  model.eval()

                  lowercase							:  Optional[int] 			=   self._prepare_for_class(snake_case	,snake_case   )
                  lowercase							:  Tuple 			=   inputs["""input_ids"""].shape[0]
                  with torch.no_grad():
                         lowercase							:  List[Any] 			=   model(**snake_case   )
                  lowercase							:  str 			=   outputs.attentions
                  self.assertEqual(len(snake_case   )	,self.model_tester.num_hidden_layers   )

                  self.assertListEqual(
                      list(attentions[0].shape[-3:]   )	,[batch_size, seq_len, config.hidden_size]	,)
                  lowercase							:  Dict 			=   len(snake_case   )

                  # Check attention is always last and order is fine
                  lowercase							:  Any 			=   True
                  lowercase							:  List[str] 			=   True
                  lowercase							:  Optional[Any] 			=   model_class(snake_case   )
                  model.to(snake_case   )
                  model.eval()

                  lowercase							:  Optional[int] 			=   self._prepare_for_class(snake_case	,snake_case   )
                  lowercase							:  Optional[Any] 			=   inputs["""input_ids"""].shape[0]
                  with torch.no_grad():
                         lowercase							:  Union[str, Any] 			=   model(**snake_case   )

                  lowercase							:  Tuple 			=   1
                  self.assertEqual(out_len + added_hidden_states	,len(snake_case   )   )

                  lowercase							:  Optional[Any] 			=   outputs.attentions

                  self.assertEqual(len(snake_case   )	,self.model_tester.num_hidden_layers   )
                  self.assertListEqual(
                      list(self_attentions[0].shape[-3:]   )	,[batch_size, seq_len, config.hidden_size]	,)
    @slow
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  lowercase							:  Dict 			=   RwkvModel.from_pretrained(snake_case   )
                  self.assertIsNotNone(snake_case   )






@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0      ,			reason="See https://github.com/huggingface/transformers/pull/24204"  )
@slow
class __snake_case	(   unittest.TestCase  ):
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   """RWKV/rwkv-4-169m-pile"""
           lowercase							:  str 			=   AutoTokenizer.from_pretrained(self.model_id   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  List[Any] 			=   """Hello my name is Jasmine and I am a newbie to the"""
           lowercase							:  List[Any] 			=   RwkvForCausalLM.from_pretrained(self.model_id   ).to(snake_case   )

           lowercase							:  Union[str, Any] 			=   self.tokenizer("""Hello my name is"""	,return_tensors="""pt"""   ).input_ids.to(snake_case   )
           lowercase							:  Optional[Any] 			=   model.generate(snake_case	,max_new_tokens=10   )
           lowercase							:  List[Any] 			=   self.tokenizer.decode(output[0].tolist()   )

           self.assertEqual(snake_case	,snake_case   )
    def    _SCREAMING_SNAKE_CASE (						self   ):



           '''simple docstring'''







           lowercase							:  Any 			=   """Hello my name is Jasmine and I am a newbie to the"""

           lowercase							:  Optional[Any] 			=   self.tokenizer("""Hello my name is"""	,return_tensors="""pt"""   ).input_ids.to(snake_case   )
           lowercase							:  Union[str, Any] 			=   RwkvForCausalLM.from_pretrained(self.model_id	,torch_dtype=torch.bfloataa   ).to(snake_case   )

           lowercase							:  Tuple 			=   model.generate(snake_case	,max_new_tokens=10   )
           lowercase							:  Optional[int] 			=   self.tokenizer.decode(output[0].tolist()   )

           self.assertEqual(snake_case	,snake_case   )
