



'''simple docstring'''

import tempfile
import unittest

from transformers import DPRConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import DPRContextEncoder, DPRQuestionEncoder, DPRReader, DPRReaderTokenizer
							from transformers.models.dpr.modeling_dpr import (
							    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
							    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
							    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Tuple=13			,_UpperCAmelCase      : Any=7			,_UpperCAmelCase      : str=False			,_UpperCAmelCase      : Any=True			,_UpperCAmelCase      : Dict=True			,_UpperCAmelCase      : str=True			,_UpperCAmelCase      : Optional[Any]=99			,_UpperCAmelCase      : List[str]=32			,_UpperCAmelCase      : Optional[int]=5			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : int=37			,_UpperCAmelCase      : List[Any]="gelu"			,_UpperCAmelCase      : Tuple=0.1			,_UpperCAmelCase      : List[str]=0.1			,_UpperCAmelCase      : Union[str, Any]=512			,_UpperCAmelCase      : Any=16			,_UpperCAmelCase      : Optional[Any]=2			,_UpperCAmelCase      : Tuple=0.02			,_UpperCAmelCase      : Dict=3			,_UpperCAmelCase      : List[str]=4			,_UpperCAmelCase      : List[Any]=None			,_UpperCAmelCase      : Dict=0			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Tuple				 =							batch_size
									_a  :   Optional[Any]				 =							seq_length
									_a  :   List[Any]				 =							is_training
									_a  :   Optional[int]				 =							use_input_mask
									_a  :   List[Any]				 =							use_token_type_ids
									_a  :   Dict				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   int				 =							hidden_size
									_a  :   Tuple				 =							num_hidden_layers
									_a  :   Optional[int]				 =							num_attention_heads
									_a  :   Any				 =							intermediate_size
									_a  :   Dict				 =							hidden_act
									_a  :   Any				 =							hidden_dropout_prob
									_a  :   str				 =							attention_probs_dropout_prob
									_a  :   Dict				 =							max_position_embeddings
									_a  :   Dict				 =							type_vocab_size
									_a  :   Union[str, Any]				 =							type_sequence_label_size
									_a  :   Optional[Any]				 =							initializer_range
									_a  :   List[Any]				 =							num_labels
									_a  :   Optional[Any]				 =							num_choices
									_a  :   str				 =							scope
									_a  :   str				 =							projection_dim







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   List[Any]				 =							None
									if self.use_input_mask:
											_a  :   Tuple				 =							random_attention_mask([self.batch_size, self.seq_length]	)

									_a  :   Union[str, Any]				 =							None
									if self.use_token_type_ids:
											_a  :   List[str]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   List[Any]				 =							None
									_a  :   Optional[Any]				 =							None
									_a  :   Any				 =							None
									if self.use_labels:
											_a  :   List[Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   Tuple				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   int				 =							ids_tensor([self.batch_size]			,self.num_choices	)

									_a  :   int				 =							self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







							def 							__lowercase    (							self      : Optional[int]	):
									return DPRConfig(
									    projection_dim=self.projection_dim			,vocab_size=self.vocab_size			,hidden_size=self.hidden_size			,num_hidden_layers=self.num_hidden_layers			,num_attention_heads=self.num_attention_heads			,intermediate_size=self.intermediate_size			,hidden_act=self.hidden_act			,hidden_dropout_prob=self.hidden_dropout_prob			,attention_probs_dropout_prob=self.attention_probs_dropout_prob			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_range=self.initializer_range			,)







							def 							__lowercase    (							self      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : int			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									_a  :   List[str]				 =							DPRContextEncoder(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Dict				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.pooler_output.shape			,(self.batch_size, self.projection_dim or self.hidden_size)	)







							def 							__lowercase    (							self      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]	):
									_a  :   List[Any]				 =							DPRQuestionEncoder(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.pooler_output.shape			,(self.batch_size, self.projection_dim or self.hidden_size)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict	):
									_a  :   Dict				 =							DPRReader(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Tuple				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,)

									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.relevance_logits.shape			,(self.batch_size,)	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   Tuple				 =							config_and_inputs
									_a  :   str				 =							{'input_ids': input_ids}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        DPRContextEncoder,
							        DPRQuestionEncoder,
							        DPRReader,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					List[str]            =      {'feature-extraction': DPRQuestionEncoder} if is_torch_available() else {}

							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					Optional[int]            =      False  # why?
							lowerCAmelCase :					List[str]            =      False
							lowerCAmelCase :					str            =      False







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   List[Any]				 =							DPRModelTester(self	)
									_a  :   Optional[Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase			,hidden_size=37	)







							def 							__lowercase    (							self      : List[str]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_context_encoder(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   str				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_question_encoder(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_reader(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()[0]

									_a  :   Optional[Any]				 =							DPRQuestionEncoder(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()

									with tempfile.TemporaryDirectory() as tmp_dirname:
											model.save_pretrained(_UpperCAmelCase	)
											_a  :   Optional[Any]				 =							DPRQuestionEncoder.from_pretrained(_UpperCAmelCase			,projection_dim=512	)

									self.assertIsNotNone(_UpperCAmelCase	)







							@slow
							def 							__lowercase    (							self      : List[str]	):
									for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a  :   List[Any]				 =							DPRContextEncoder.from_pretrained(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)

									for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a  :   Union[str, Any]				 =							DPRContextEncoder.from_pretrained(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)

									for model_name in DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a  :   Tuple				 =							DPRQuestionEncoder.from_pretrained(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)

									for model_name in DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a  :   Optional[int]				 =							DPRReader.from_pretrained(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)

@require_torch
class 			__magic_name__		(					unittest.TestCase     ):







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Tuple				 =							DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'			,return_dict=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)

									_a  :   Optional[Any]				 =							torch.tensor(
									    [[101, 7592, 1010, 2003, 2026, 3899, 10140, 1029, 102]]			,dtype=torch.long			,device=_UpperCAmelCase	)  # [CLS] hello, is my dog cute? [SEP]
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase	)[0]  # embedding shape = (1, 768)
									# compare the actual values for a slice.
									_a  :   List[str]				 =							torch.tensor(
									    [
									        [
									            0.03_23_62_53,
									            0.12_75_33_35,
									            0.16_81_85_09,
									            0.00_27_97_86,
									            0.3_89_69_33,
									            0.24_26_49_45,
									            0.2_17_89_71,
									            -0.02_33_52_27,
									            -0.08_48_19_59,
									            -0.14_32_41_17,
									        ]
									    ]			,dtype=torch.float			,device=_UpperCAmelCase			,)
									self.assertTrue(torch.allclose(output[:, :10]			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : Tuple	):
									_a  :   List[str]				 =							DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base'	)
									_a  :   Tuple				 =							DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base'	)
									model.to(_UpperCAmelCase	)

									_a  :   Any				 =							tokenizer(
									    questions='What is love ?'			,titles='Haddaway'			,texts='What Is Love is a song recorded by the artist Haddaway'			,padding=_UpperCAmelCase			,return_tensors='pt'			,)
									encoded_inputs.to(_UpperCAmelCase	)

									_a  :   int				 =							model(**_UpperCAmelCase	)

									# compare the actual values for a slice.
									_a  :   Tuple				 =							torch.tensor(
									    [[-10.30_05, -10.77_65, -11.48_72, -11.68_41, -11.93_12, -10.30_02, -9.85_44, -11.73_78, -12.08_21, -10.29_75]]			,dtype=torch.float			,device=_UpperCAmelCase			,)

									_a  :   Dict				 =							torch.tensor(
									    [[-11.06_84, -11.70_41, -11.53_97, -10.34_65, -10.87_91, -6.84_43, -11.99_59, -11.03_64, -10.00_96, -6.84_05]]			,dtype=torch.float			,device=_UpperCAmelCase			,)
									self.assertTrue(torch.allclose(outputs.start_logits[:, :10]			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(outputs.end_logits[:, :10]			,_UpperCAmelCase			,atol=1E-4	)	)









'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : int=13			,_UpperCAmelCase      : List[str]=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : List[Any]=True			,_UpperCAmelCase      : Tuple=True			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=99			,_UpperCAmelCase      : Optional[Any]=[1, 1, 2]			,_UpperCAmelCase      : Optional[Any]=1			,_UpperCAmelCase      : Any=32			,_UpperCAmelCase      : Optional[Any]=4			,_UpperCAmelCase      : Optional[int]=8			,_UpperCAmelCase      : str=37			,_UpperCAmelCase      : Tuple="gelu_new"			,_UpperCAmelCase      : Optional[int]=0.1			,_UpperCAmelCase      : List[Any]=0.1			,_UpperCAmelCase      : Dict=0.0			,_UpperCAmelCase      : str=512			,_UpperCAmelCase      : Any=3			,_UpperCAmelCase      : int=0.02			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : Optional[Any]=None			,_UpperCAmelCase      : Optional[int]=False			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Optional[Any]				 =							batch_size
									_a  :   Union[str, Any]				 =							seq_length
									_a  :   int				 =							is_training
									_a  :   List[Any]				 =							use_input_mask
									_a  :   Dict				 =							use_token_type_ids
									_a  :   List[str]				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   Union[str, Any]				 =							block_sizes
									_a  :   int				 =							num_decoder_layers
									_a  :   List[str]				 =							d_model
									_a  :   Optional[Any]				 =							n_head
									_a  :   int				 =							d_head
									_a  :   Union[str, Any]				 =							d_inner
									_a  :   Any				 =							hidden_act
									_a  :   int				 =							hidden_dropout
									_a  :   str				 =							attention_dropout
									_a  :   Any				 =							activation_dropout
									_a  :   List[Any]				 =							max_position_embeddings
									_a  :   Optional[int]				 =							type_vocab_size
									_a  :   Optional[int]				 =							2
									_a  :   Dict				 =							num_labels
									_a  :   List[str]				 =							num_choices
									_a  :   List[str]				 =							scope
									_a  :   Any				 =							initializer_std

									# Used in the tests to check the size of the first attention layer
									_a  :   List[Any]				 =							n_head
									# Used in the tests to check the size of the first hidden state
									_a  :   List[Any]				 =							self.d_model
									# Used in the tests to check the number of output hidden states/attentions
									_a  :   Tuple				 =							sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
									# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
									# the last hidden state of the first block (which is the first hidden state of the decoder).
									if not base:
											_a  :   Tuple				 =							self.num_hidden_layers + 2







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   Dict				 =							None
									if self.use_input_mask:
											_a  :   str				 =							ids_tensor([self.batch_size, self.seq_length]			,vocab_size=2	)

									_a  :   Any				 =							None
									if self.use_token_type_ids:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   str				 =							None
									_a  :   Optional[int]				 =							None
									_a  :   str				 =							None
									if self.use_labels:
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   List[Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   List[str]				 =							ids_tensor([self.batch_size]			,self.num_choices	)
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,1	)

									_a  :   Dict				 =							self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    fake_token_labels,
									)







							def 							__lowercase    (							self      : Any	):
									return FunnelConfig(
									    vocab_size=self.vocab_size			,block_sizes=self.block_sizes			,num_decoder_layers=self.num_decoder_layers			,d_model=self.d_model			,n_head=self.n_head			,d_head=self.d_head			,d_inner=self.d_inner			,hidden_act=self.hidden_act			,hidden_dropout=self.hidden_dropout			,attention_dropout=self.attention_dropout			,activation_dropout=self.activation_dropout			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_std=self.initializer_std			,)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,):
									_a  :   int				 =							FunnelModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   int				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Optional[Any]				 =							False
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Any			,):
									_a  :   int				 =							FunnelBaseModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Tuple				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)

									_a  :   Dict				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 3, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   List[Any]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,):
									_a  :   Union[str, Any]				 =							self.num_labels
									_a  :   Any				 =							FunnelForPreTraining(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,):
									_a  :   Tuple				 =							FunnelForMaskedLM(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   List[str]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   List[str]				 =							self.num_labels
									_a  :   Dict				 =							FunnelForSequenceClassification(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : int			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   int				 =							self.num_choices
									_a  :   Dict				 =							FunnelForMultipleChoice(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							input_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   List[Any]				 =							token_type_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Tuple				 =							input_mask.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase			,)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,):
									_a  :   Any				 =							self.num_labels
									_a  :   int				 =							FunnelForTokenClassification(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[Any]			,):
									_a  :   List[str]				 =							FunnelForQuestionAnswering(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,start_positions=_UpperCAmelCase			,end_positions=_UpperCAmelCase			,)
									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Tuple				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   List[str]				 =							config_and_inputs
									_a  :   int				 =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					Optional[Any]            =      (
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[int]=False	):
									_a  :   List[Any]				 =							super()._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)

									if return_labels:
											if model_class in get_values(_UpperCAmelCase	):
													_a  :   Optional[int]				 =							torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=_UpperCAmelCase	)
									return inputs_dict







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   int				 =							FunnelModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Union[str, Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   str				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Optional[int]            =      False
							lowerCAmelCase :					Optional[Any]            =      False
							lowerCAmelCase :					Optional[Any]            =      (
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							FunnelModelTester(self			,base=_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_base_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a ,      _a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   str				 =							True

									for model_class in self.all_model_classes:
											if model_class.__name__ == "FunnelBaseModel":
													continue
											_a  :   Optional[int]				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.train()
											_a  :   Dict				 =							self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)
											_a  :   Dict				 =							model(**_UpperCAmelCase	).loss
											loss.backward()







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   Tuple				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Any	):
									_a  :   List[Any]				 =							13
									_a  :   Dict				 =							7
									_a  :   Union[str, Any]				 =							torch.arange(0			,batch_size * sequence_length	).long().reshape(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   Tuple				 =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
									_a  :   List[Any]				 =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

									_a  :   Tuple				 =							FunnelModel.from_pretrained('sgugger/funnel-random-tiny'	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Optional[int]				 =							torch.tensor(23_44.83_52	)
									_a  :   Union[str, Any]				 =							torch.tensor(0.80_52	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)

									_a  :   Tuple				 =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Union[str, Any]				 =							torch.tensor(23_43.84_25	)
									_a  :   Tuple				 =							torch.tensor(0.80_49	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Union[str, Any]				 =							FunnelTokenizer.from_pretrained('huggingface/funnel-small'	)
									_a  :   Tuple				 =							FunnelModel.from_pretrained('huggingface/funnel-small'	)
									_a  :   Union[str, Any]				 =							tokenizer('Hello! I am the Funnel Transformer model.'			,return_tensors='pt'	)
									_a  :   str				 =							model(**_UpperCAmelCase	)[0]

									_a  :   str				 =							torch.tensor(2_35.72_46	)
									_a  :   List[str]				 =							torch.tensor(0.02_56	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)




