





'''simple docstring'''






import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						)




class 				UpperCAmelCase  :




						def __init__(    self    :				Optional[int]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				Tuple=13    ,    __snake_case    :				Tuple=7    ,    __snake_case    :				Any=True    ,    __snake_case    :				Optional[int]=True    ,    __snake_case    :				Tuple=True    ,    __snake_case    :				Optional[Any]=True    ,    __snake_case    :				Union[str, Any]=99    ,    __snake_case    :				List[Any]=[1, 1, 2]    ,    __snake_case    :				int=1    ,    __snake_case    :				List[Any]=32    ,    __snake_case    :				Tuple=4    ,    __snake_case    :				Tuple=8    ,    __snake_case    :				Optional[int]=37    ,    __snake_case    :				int="gelu_new"    ,    __snake_case    :				Dict=0.1    ,    __snake_case    :				List[str]=0.1    ,    __snake_case    :				str=0.0    ,    __snake_case    :				int=5_12    ,    __snake_case    :				Dict=3    ,    __snake_case    :				Union[str, Any]=0.02    ,    __snake_case    :				Union[str, Any]=3    ,    __snake_case    :				Tuple=4    ,    __snake_case    :				Optional[Any]=None    ,    __snake_case    :				Tuple=False    ,    )       ->   Any:
												_lowerCAmelCase							  =							parent
												_lowerCAmelCase							  =							batch_size
												_lowerCAmelCase							  =							seq_length
												_lowerCAmelCase							  =							is_training
												_lowerCAmelCase							  =							use_input_mask
												_lowerCAmelCase							  =							use_token_type_ids
												_lowerCAmelCase							  =							use_labels
												_lowerCAmelCase							  =							vocab_size
												_lowerCAmelCase							  =							block_sizes
												_lowerCAmelCase							  =							num_decoder_layers
												_lowerCAmelCase							  =							d_model
												_lowerCAmelCase							  =							n_head
												_lowerCAmelCase							  =							d_head
												_lowerCAmelCase							  =							d_inner
												_lowerCAmelCase							  =							hidden_act
												_lowerCAmelCase							  =							hidden_dropout
												_lowerCAmelCase							  =							attention_dropout
												_lowerCAmelCase							  =							activation_dropout
												_lowerCAmelCase							  =							max_position_embeddings
												_lowerCAmelCase							  =							type_vocab_size
												_lowerCAmelCase							  =							2
												_lowerCAmelCase							  =							num_labels
												_lowerCAmelCase							  =							num_choices
												_lowerCAmelCase							  =							scope
												_lowerCAmelCase							  =							initializer_std

												# Used in the tests to check the size of the first attention layer
												_lowerCAmelCase							  =							n_head
												# Used in the tests to check the size of the first hidden state
												_lowerCAmelCase							  =							self.d_model
												# Used in the tests to check the number of output hidden states/attentions
												_lowerCAmelCase							  =							sum(self.block_sizes						) + (0 if base else self.num_decoder_layers)
												# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
												# the last hidden state of the first block (which is the first hidden state of the decoder).
												if not base:
																		_lowerCAmelCase							  =							self.num_hidden_layers + 2




						def        lowercase__					(    self    :				int						)       ->   Optional[int]:
												_lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size						)

												_lowerCAmelCase							  =							None
												if self.use_input_mask:
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    vocab_size=2						)

												_lowerCAmelCase							  =							None
												if self.use_token_type_ids:
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size						)

												_lowerCAmelCase							  =							None
												_lowerCAmelCase							  =							None
												_lowerCAmelCase							  =							None
												if self.use_labels:
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size]    ,    self.type_sequence_label_size						)
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.num_labels						)
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size]    ,    self.num_choices						)
																		_lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    1						)

												_lowerCAmelCase							  =							self.get_config()

												return (
												    config,
												    input_ids,
												    token_type_ids,
												    input_mask,
												    sequence_labels,
												    token_labels,
												    choice_labels,
												    fake_token_labels,
												)




						def        lowercase__					(    self    :				Optional[int]						)       ->   Optional[Any]:
												return FunnelConfig(
												    vocab_size=self.vocab_size    ,    block_sizes=self.block_sizes    ,    num_decoder_layers=self.num_decoder_layers    ,    d_model=self.d_model    ,    n_head=self.n_head    ,    d_head=self.d_head    ,    d_inner=self.d_inner    ,    hidden_act=self.hidden_act    ,    hidden_dropout=self.hidden_dropout    ,    attention_dropout=self.attention_dropout    ,    activation_dropout=self.activation_dropout    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_std=self.initializer_std    ,    )




						def        lowercase__					(    self    :				Optional[Any]    ,    __snake_case    :				str    ,    __snake_case    :				List[str]    ,    __snake_case    :				Union[str, Any]    ,    __snake_case    :				Tuple    ,    __snake_case    :				str    ,    __snake_case    :				List[Any]    ,    __snake_case    :				Optional[int]    ,    __snake_case    :				Union[str, Any]    ,    )       ->   Optional[int]:
												_lowerCAmelCase							  =							FunnelModel(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)						)

												_lowerCAmelCase							  =							False
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)						)

												_lowerCAmelCase							  =							False
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)						)




						def        lowercase__					(    self    :				Union[str, Any]    ,    __snake_case    :				List[str]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				Optional[Any]    ,    __snake_case    :				str    ,    __snake_case    :				Any    ,    __snake_case    :				Optional[Any]    ,    __snake_case    :				Any    ,    )       ->   str:
												_lowerCAmelCase							  =							FunnelBaseModel(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 2, self.d_model)						)

												_lowerCAmelCase							  =							False
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 3, self.d_model)						)

												_lowerCAmelCase							  =							False
												_lowerCAmelCase							  =							model(lowerCAmelCase__						)
												self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 2, self.d_model)						)




						def        lowercase__					(    self    :				Tuple    ,    __snake_case    :				Union[str, Any]    ,    __snake_case    :				str    ,    __snake_case    :				Dict    ,    __snake_case    :				Any    ,    __snake_case    :				List[str]    ,    __snake_case    :				Any    ,    __snake_case    :				int    ,    __snake_case    :				Optional[int]    ,    )       ->   Optional[int]:
												_lowerCAmelCase							  =							self.num_labels
												_lowerCAmelCase							  =							FunnelForPreTraining(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    labels=lowerCAmelCase__						)
												self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length)						)




						def        lowercase__					(    self    :				Tuple    ,    __snake_case    :				Union[str, Any]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				str    ,    __snake_case    :				List[Any]    ,    __snake_case    :				List[str]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				Union[str, Any]    ,    )       ->   Tuple:
												_lowerCAmelCase							  =							FunnelForMaskedLM(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    labels=lowerCAmelCase__						)
												self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length, self.vocab_size)						)




						def        lowercase__					(    self    :				List[Any]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				int    ,    __snake_case    :				Dict    ,    __snake_case    :				Optional[Any]    ,    __snake_case    :				Optional[int]    ,    __snake_case    :				str    ,    __snake_case    :				Any    ,    __snake_case    :				Dict    ,    )       ->   Dict:
												_lowerCAmelCase							  =							self.num_labels
												_lowerCAmelCase							  =							FunnelForSequenceClassification(lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    labels=lowerCAmelCase__						)
												self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.num_labels)						)




						def        lowercase__					(    self    :				int    ,    __snake_case    :				List[str]    ,    __snake_case    :				List[str]    ,    __snake_case    :				List[Any]    ,    __snake_case    :				str    ,    __snake_case    :				int    ,    __snake_case    :				Tuple    ,    __snake_case    :				List[Any]    ,    __snake_case    :				str    ,    )       ->   Optional[Any]:
												_lowerCAmelCase							  =							self.num_choices
												_lowerCAmelCase							  =							FunnelForMultipleChoice(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							input_ids.unsqueeze(1						).expand(-1    ,    self.num_choices    ,    -1						).contiguous()
												_lowerCAmelCase							  =							token_type_ids.unsqueeze(1						).expand(-1    ,    self.num_choices    ,    -1						).contiguous()
												_lowerCAmelCase							  =							input_mask.unsqueeze(1						).expand(-1    ,    self.num_choices    ,    -1						).contiguous()
												_lowerCAmelCase							  =							model(
												    lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    labels=lowerCAmelCase__    ,    )
												self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.num_choices)						)




						def        lowercase__					(    self    :				Any    ,    __snake_case    :				Any    ,    __snake_case    :				List[Any]    ,    __snake_case    :				str    ,    __snake_case    :				str    ,    __snake_case    :				List[str]    ,    __snake_case    :				Any    ,    __snake_case    :				int    ,    __snake_case    :				Dict    ,    )       ->   Tuple:
												_lowerCAmelCase							  =							self.num_labels
												_lowerCAmelCase							  =							FunnelForTokenClassification(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    labels=lowerCAmelCase__						)
												self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length, self.num_labels)						)




						def        lowercase__					(    self    :				Optional[int]    ,    __snake_case    :				Dict    ,    __snake_case    :				Dict    ,    __snake_case    :				Tuple    ,    __snake_case    :				List[str]    ,    __snake_case    :				str    ,    __snake_case    :				List[Any]    ,    __snake_case    :				Optional[int]    ,    __snake_case    :				Optional[Any]    ,    )       ->   str:
												_lowerCAmelCase							  =							FunnelForQuestionAnswering(config=lowerCAmelCase__						)
												model.to(lowerCAmelCase__						)
												model.eval()
												_lowerCAmelCase							  =							model(
												    lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__    ,    start_positions=lowerCAmelCase__    ,    end_positions=lowerCAmelCase__    ,    )
												self.parent.assertEqual(result.start_logits.shape    ,    (self.batch_size, self.seq_length)						)
												self.parent.assertEqual(result.end_logits.shape    ,    (self.batch_size, self.seq_length)						)




						def        lowercase__					(    self    :				Tuple						)       ->   Optional[int]:
												_lowerCAmelCase							  =							self.prepare_config_and_inputs()
												(
												    _lowerCAmelCase							
												)							  =							config_and_inputs
												_lowerCAmelCase							  =							{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
												return config, inputs_dict




@require_torch
class 				UpperCAmelCase  (    UpperCamelCase__							,	UpperCamelCase__							,	unittest.TestCase     ):
						_lowercase:    Optional[Any]		  =     False
						_lowercase:    int		  =     False
						_lowercase:    Optional[int]		  =     (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowercase:    List[str]		  =     (
						    {
						        """feature-extraction""": (FunnelBaseModel, FunnelModel),
						        """fill-mask""": FunnelForMaskedLM,
						        """question-answering""": FunnelForQuestionAnswering,
						        """text-classification""": FunnelForSequenceClassification,
						        """token-classification""": FunnelForTokenClassification,
						        """zero-shot""": FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)




						def        lowercase__					(    self    :				Any    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				int=False						)       ->   List[str]:
												_lowerCAmelCase							  =							super()._prepare_for_class(lowerCAmelCase__    ,    lowerCAmelCase__    ,    return_labels=lowerCAmelCase__						)

												if return_labels:
																		if model_class in get_values(lowerCAmelCase__						):
																								_lowerCAmelCase							  =							torch.zeros(
																								    (self.model_tester.batch_size, self.model_tester.seq_length)    ,    dtype=torch.long    ,    device=lowerCAmelCase__						)
												return inputs_dict




						def        lowercase__					(    self    :				Optional[int]						)       ->   List[str]:
												_lowerCAmelCase							  =							FunnelModelTester(self						)
												_lowerCAmelCase							  =							ConfigTester(self    ,    config_class=lowerCAmelCase__						)




						def        lowercase__					(    self    :				List[str]						)       ->   List[str]:
												self.config_tester.run_common_tests()




						def        lowercase__					(    self    :				List[Any]						)       ->   Optional[int]:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Dict						)       ->   List[str]:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Any						)       ->   Optional[int]:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Union[str, Any]						)       ->   List[Any]:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Dict						)       ->   Tuple:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				List[Any]    ,    __snake_case    :				Any						)       ->   Optional[Any]:




												if hasattr(lowerCAmelCase__    ,    """weight"""						) and module.weight is not None:
																		module.weight.data.fill_(3						)
												if hasattr(lowerCAmelCase__    ,    """bias"""						) and module.bias is not None:
																		module.bias.data.fill_(3						)

												for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																		if hasattr(lowerCAmelCase__    ,    lowerCAmelCase__						) and getattr(lowerCAmelCase__    ,    lowerCAmelCase__						) is not None:
																								_lowerCAmelCase							  =							getattr(lowerCAmelCase__    ,    lowerCAmelCase__						)
																								weight.data.fill_(3						)




@require_torch
class 				UpperCAmelCase  (    UpperCamelCase__							,	unittest.TestCase     ):
						_lowercase:    Tuple		  =     False
						_lowercase:    List[str]		  =     False
						_lowercase:    str		  =     (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)




						def        lowercase__					(    self    :				List[str]						)       ->   List[Any]:
												_lowerCAmelCase							  =							FunnelModelTester(self    ,    base=lowerCAmelCase__						)
												_lowerCAmelCase							  =							ConfigTester(self    ,    config_class=lowerCAmelCase__						)




						def        lowercase__					(    self    :				Any						)       ->   List[str]:
												self.config_tester.run_common_tests()




						def        lowercase__					(    self    :				str						)       ->   Tuple:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_base_model(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Optional[Any]						)       ->   Any:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				str						)       ->   int:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__						)




						def        lowercase__					(    self    :				Optional[int]						)       ->   int:
												_lowerCAmelCase							  =							self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase							  =							True

												for model_class in self.all_model_classes:
																		if model_class.__name__ == "FunnelBaseModel":
																								continue
																		_lowerCAmelCase							  =							model_class(lowerCAmelCase__						)
																		model.to(lowerCAmelCase__						)
																		model.train()
																		_lowerCAmelCase							  =							self._prepare_for_class(lowerCAmelCase__    ,    lowerCAmelCase__    ,    return_labels=lowerCAmelCase__						)
																		_lowerCAmelCase							  =							model(**lowerCAmelCase__						).loss
																		loss.backward()




						def        lowercase__					(    self    :				Dict    ,    __snake_case    :				str						)       ->   str:




												if hasattr(lowerCAmelCase__    ,    """weight"""						) and module.weight is not None:
																		module.weight.data.fill_(3						)
												if hasattr(lowerCAmelCase__    ,    """bias"""						) and module.bias is not None:
																		module.bias.data.fill_(3						)

												for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																		if hasattr(lowerCAmelCase__    ,    lowerCAmelCase__						) and getattr(lowerCAmelCase__    ,    lowerCAmelCase__						) is not None:
																								_lowerCAmelCase							  =							getattr(lowerCAmelCase__    ,    lowerCAmelCase__						)
																								weight.data.fill_(3						)




@require_torch
@require_sentencepiece
@require_tokenizers
class 				UpperCAmelCase  (    unittest.TestCase     ):




						def        lowercase__					(    self    :				Any						)       ->   Optional[Any]:
												_lowerCAmelCase							  =							13
												_lowerCAmelCase							  =							7
												_lowerCAmelCase							  =							torch.arange(0    ,    batch_size * sequence_length						).long().reshape(lowerCAmelCase__    ,    lowerCAmelCase__						)
												_lowerCAmelCase							  =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
												_lowerCAmelCase							  =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]						)

												_lowerCAmelCase							  =							FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)[0].abs()

												_lowerCAmelCase							  =							torch.tensor(23_44.83_52						)
												_lowerCAmelCase							  =							torch.tensor(0.80_52						)
												self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)
												self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)

												_lowerCAmelCase							  =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]						)
												_lowerCAmelCase							  =							model(lowerCAmelCase__    ,    attention_mask=lowerCAmelCase__    ,    token_type_ids=lowerCAmelCase__						)[0].abs()

												_lowerCAmelCase							  =							torch.tensor(23_43.84_25						)
												_lowerCAmelCase							  =							torch.tensor(0.80_49						)
												self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)
												self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)




						@slow
						def        lowercase__					(    self    :				List[Any]						)       ->   str:
												_lowerCAmelCase							  =							FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""						)
												_lowerCAmelCase							  =							FunnelModel.from_pretrained("""huggingface/funnel-small"""						)
												_lowerCAmelCase							  =							tokenizer("""Hello! I am the Funnel Transformer model."""    ,    return_tensors="""pt"""						)
												_lowerCAmelCase							  =							model(**lowerCAmelCase__						)[0]

												_lowerCAmelCase							  =							torch.tensor(2_35.72_46						)
												_lowerCAmelCase							  =							torch.tensor(0.02_56						)
												self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)
												self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase__    ,    atol=1E-4						)						)













'''simple docstring'''






import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      from transformers.models.electra.modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
      )




class 				UpperCAmelCase  (    unittest.TestCase     ):




      def __init__(    self    :				str    ,    __snake_case    :				str    ,    __snake_case    :				str=13    ,    __snake_case    :				Optional[Any]=7    ,    __snake_case    :				str=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=99    ,    __snake_case    :				Dict=24    ,    __snake_case    :				Any=32    ,    __snake_case    :				List[str]=5    ,    __snake_case    :				Any=4    ,    __snake_case    :				Optional[Any]=37    ,    __snake_case    :				Union[str, Any]="gelu"    ,    __snake_case    :				Dict=0.1    ,    __snake_case    :				int=0.1    ,    __snake_case    :				Dict=5_12    ,    __snake_case    :				List[Any]=16    ,    __snake_case    :				Dict=2    ,    __snake_case    :				str=0.02    ,    __snake_case    :				Any=4    ,    )       ->   Union[str, Any]:
            _lowerCAmelCase							  =							parent
            _lowerCAmelCase							  =							batch_size
            _lowerCAmelCase							  =							seq_length
            _lowerCAmelCase							  =							is_training
            _lowerCAmelCase							  =							use_attention_mask
            _lowerCAmelCase							  =							use_token_type_ids
            _lowerCAmelCase							  =							use_labels
            _lowerCAmelCase							  =							vocab_size
            _lowerCAmelCase							  =							hidden_size
            _lowerCAmelCase							  =							embedding_size
            _lowerCAmelCase							  =							num_hidden_layers
            _lowerCAmelCase							  =							num_attention_heads
            _lowerCAmelCase							  =							intermediate_size
            _lowerCAmelCase							  =							hidden_act
            _lowerCAmelCase							  =							hidden_dropout_prob
            _lowerCAmelCase							  =							attention_probs_dropout_prob
            _lowerCAmelCase							  =							max_position_embeddings
            _lowerCAmelCase							  =							type_vocab_size
            _lowerCAmelCase							  =							type_sequence_label_size
            _lowerCAmelCase							  =							initializer_range
            _lowerCAmelCase							  =							num_choices




      def        lowercase__					(    self    :				Optional[Any]						)       ->   Optional[Any]:
            _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size						)

            _lowerCAmelCase							  =							None
            if self.use_attention_mask:
                  _lowerCAmelCase							  =							random_attention_mask([self.batch_size, self.seq_length]						)

            _lowerCAmelCase							  =							None
            if self.use_token_type_ids:
                  _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size						)

            _lowerCAmelCase							  =							ElectraConfig(
                vocab_size=self.vocab_size    ,    hidden_size=self.hidden_size    ,    embedding_size=self.embedding_size    ,    num_hidden_layers=self.num_hidden_layers    ,    num_attention_heads=self.num_attention_heads    ,    intermediate_size=self.intermediate_size    ,    hidden_act=self.hidden_act    ,    hidden_dropout_prob=self.hidden_dropout_prob    ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_range=self.initializer_range    ,    )

            return config, input_ids, token_type_ids, attention_mask




      def        lowercase__					(    self    :				Any						)       ->   Optional[int]:
            _lowerCAmelCase							  =							self.prepare_config_and_inputs()
            _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase							  =							config_and_inputs
            _lowerCAmelCase							  =							{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
            return config, inputs_dict




@require_flax
class 				UpperCAmelCase  (    snake_case_							,	unittest.TestCase     ):
      _lowercase:    str		  =     True

      _lowercase:    Optional[Any]		  =     (
          (
              FlaxElectraModel,
              FlaxElectraForCausalLM,
              FlaxElectraForMaskedLM,
              FlaxElectraForPreTraining,
              FlaxElectraForTokenClassification,
              FlaxElectraForQuestionAnswering,
              FlaxElectraForMultipleChoice,
              FlaxElectraForSequenceClassification,
          )
          if is_flax_available()
          else ()
      )




      def        lowercase__					(    self    :				Tuple						)       ->   Optional[int]:
            _lowerCAmelCase							  =							FlaxElectraModelTester(self						)




      @slow
      def        lowercase__					(    self    :				List[Any]						)       ->   str:
            for model_class_name in self.all_model_classes:
                  if model_class_name == FlaxElectraForMaskedLM:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-generator"""						)
                  else:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-discriminator"""						)
                  _lowerCAmelCase							  =							model(np.ones((1, 1)						)						)
                  self.assertIsNotNone(__snake_case						)






