



'''simple docstring'''

import inspect
import unittest

from transformers import SwinaSRConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import SwinaSRForImageSuperResolution, SwinaSRModel
							from transformers.models.swinasr.modeling_swinasr import SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
							from PIL import Image

							from transformers import SwinaSRImageProcessor

class 			__magic_name__		:







							def __init__(							self      : List[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any=13			,_UpperCAmelCase      : Dict=32			,_UpperCAmelCase      : str=1			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=16			,_UpperCAmelCase      : List[Any]=[1, 2, 1]			,_UpperCAmelCase      : List[Any]=[2, 2, 4]			,_UpperCAmelCase      : str=2			,_UpperCAmelCase      : Dict=2.0			,_UpperCAmelCase      : Union[str, Any]=True			,_UpperCAmelCase      : Any=0.0			,_UpperCAmelCase      : Any=0.0			,_UpperCAmelCase      : List[str]=0.1			,_UpperCAmelCase      : str="gelu"			,_UpperCAmelCase      : Tuple=False			,_UpperCAmelCase      : Dict=True			,_UpperCAmelCase      : Tuple=0.02			,_UpperCAmelCase      : Any=1E-5			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=None			,_UpperCAmelCase      : Dict=False			,_UpperCAmelCase      : Optional[Any]=2			,):
									_a  :   Dict				 =							parent
									_a  :   Optional[int]				 =							batch_size
									_a  :   Union[str, Any]				 =							image_size
									_a  :   int				 =							patch_size
									_a  :   Optional[int]				 =							num_channels
									_a  :   str				 =							embed_dim
									_a  :   int				 =							depths
									_a  :   List[str]				 =							num_heads
									_a  :   Union[str, Any]				 =							window_size
									_a  :   Optional[Any]				 =							mlp_ratio
									_a  :   Union[str, Any]				 =							qkv_bias
									_a  :   str				 =							hidden_dropout_prob
									_a  :   Dict				 =							attention_probs_dropout_prob
									_a  :   Any				 =							drop_path_rate
									_a  :   List[Any]				 =							hidden_act
									_a  :   Union[str, Any]				 =							use_absolute_embeddings
									_a  :   Optional[Any]				 =							patch_norm
									_a  :   Tuple				 =							layer_norm_eps
									_a  :   Tuple				 =							initializer_range
									_a  :   List[Any]				 =							is_training
									_a  :   List[str]				 =							scope
									_a  :   Union[str, Any]				 =							use_labels
									_a  :   Any				 =							upscale

									# here we set some attributes to make tests pass
									_a  :   Union[str, Any]				 =							len(_UpperCAmelCase	)
									_a  :   Optional[int]				 =							embed_dim
									_a  :   Any				 =							(image_size // patch_size) ** 2







							def 							__lowercase    (							self      : Dict	):
									_a  :   Optional[Any]				 =							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

									_a  :   List[Any]				 =							None
									if self.use_labels:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)

									_a  :   Any				 =							self.get_config()

									return config, pixel_values, labels







							def 							__lowercase    (							self      : Optional[int]	):
									return SwinaSRConfig(
									    image_size=self.image_size			,patch_size=self.patch_size			,num_channels=self.num_channels			,embed_dim=self.embed_dim			,depths=self.depths			,num_heads=self.num_heads			,window_size=self.window_size			,mlp_ratio=self.mlp_ratio			,qkv_bias=self.qkv_bias			,hidden_dropout_prob=self.hidden_dropout_prob			,attention_probs_dropout_prob=self.attention_probs_dropout_prob			,drop_path_rate=self.drop_path_rate			,hidden_act=self.hidden_act			,use_absolute_embeddings=self.use_absolute_embeddings			,path_norm=self.patch_norm			,layer_norm_eps=self.layer_norm_eps			,initializer_range=self.initializer_range			,upscale=self.upscale			,)







							def 							__lowercase    (							self      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Union[str, Any]	):
									_a  :   List[Any]				 =							SwinaSRModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase	)

									self.parent.assertEqual(
									    result.last_hidden_state.shape			,(self.batch_size, self.embed_dim, self.image_size, self.image_size)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[int]	):
									_a  :   List[str]				 =							SwinaSRForImageSuperResolution(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase	)

									_a  :   str				 =							self.image_size * self.upscale
									self.parent.assertEqual(
									    result.reconstruction.shape			,(self.batch_size, self.num_channels, expected_image_size, expected_image_size)	)







							def 							__lowercase    (							self      : str	):
									_a  :   Optional[int]				 =							self.prepare_config_and_inputs()
									_a ,      _a ,      _a  :   int				 =							config_and_inputs
									_a  :   Optional[Any]				 =							{'pixel_values': pixel_values}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					List[str]            =      (SwinaSRModel, SwinaSRForImageSuperResolution) if is_torch_available() else ()
							lowerCAmelCase :					Any            =      {'feature-extraction': SwinaSRModel} if is_torch_available() else {}

							lowerCAmelCase :					Union[str, Any]            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      False
							lowerCAmelCase :					Dict            =      False
							lowerCAmelCase :					Optional[int]            =      False







							def 							__lowercase    (							self      : Dict	):
									_a  :   str				 =							SwinaSRModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase			,embed_dim=37	)







							def 							__lowercase    (							self      : str	):
									self.config_tester.create_and_test_config_to_json_string()
									self.config_tester.create_and_test_config_to_json_file()
									self.config_tester.create_and_test_config_from_and_save_pretrained()
									self.config_tester.create_and_test_config_with_num_labels()
									self.config_tester.check_config_can_be_init_without_params()
									self.config_tester.check_config_arguments_init()







							def 							__lowercase    (							self      : Optional[int]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Union[str, Any]	):
									_a  :   str				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_super_resolution(*_UpperCAmelCase	)







							@unittest.skip(reason='Got `CUDA error: misaligned address` with PyTorch 2.0.0.'	)
							def 							__lowercase    (							self      : Dict	):
									pass







							@unittest.skip(reason='Swin2SR does not use inputs_embeds'	)
							def 							__lowercase    (							self      : Union[str, Any]	):
									pass







							@unittest.skip(reason='Swin2SR does not support training yet'	)
							def 							__lowercase    (							self      : Dict	):
									pass







							@unittest.skip(reason='Swin2SR does not support training yet'	)
							def 							__lowercase    (							self      : List[Any]	):
									pass







							def 							__lowercase    (							self      : Dict	):
									_a ,      _a  :   Any				 =							self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											_a  :   Optional[Any]				 =							model_class(_UpperCAmelCase	)
											self.assertIsInstance(model.get_input_embeddings()			,(nn.Module)	)
											_a  :   List[str]				 =							model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(_UpperCAmelCase			,nn.Linear	)	)







							def 							__lowercase    (							self      : Any	):
									_a ,      _a  :   List[str]				 =							self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											_a  :   List[Any]				 =							model_class(_UpperCAmelCase	)
											_a  :   Dict				 =							inspect.signature(model.forward	)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											_a  :   List[str]				 =							[*signature.parameters.keys()]

											_a  :   int				 =							['pixel_values']
											self.assertListEqual(arg_names[:1]			,_UpperCAmelCase	)







							@slow
							def 							__lowercase    (							self      : List[str]	):
									for model_name in SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a  :   Dict				 =							SwinaSRModel.from_pretrained(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)







							def 							__lowercase    (							self      : Dict	):
									_a ,      _a  :   List[str]				 =							self.model_tester.prepare_config_and_inputs_for_common()

									_a  :   str				 =							_config_zero_init(_UpperCAmelCase	)
									for model_class in self.all_model_classes:
											_a  :   Any				 =							model_class(config=_UpperCAmelCase	)
											for name, param in model.named_parameters():
													if "logit_scale" in name:
															continue
													if param.requires_grad:
															self.assertIn(
															    ((param.data.mean() * 1E9).round() / 1E9).item()			,[0.0, 1.0]			,msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""			,)







							def 							__lowercase    (							self      : int	):
									_a ,      _a  :   str				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   List[Any]				 =							True

									for model_class in self.all_model_classes:
											_a  :   Optional[int]				 =							True
											_a  :   List[str]				 =							False
											_a  :   Optional[int]				 =							True
											_a  :   Tuple				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.eval()
											with torch.no_grad():
													_a  :   Optional[int]				 =							model(**self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase	)	)
											_a  :   int				 =							outputs.attentions
											_a  :   str				 =							len(self.model_tester.depths	)
											self.assertEqual(len(_UpperCAmelCase	)			,_UpperCAmelCase	)

											# check that output_attentions also work using config
											del inputs_dict["output_attentions"]
											_a  :   List[str]				 =							True
											_a  :   Tuple				 =							config.window_size**2
											_a  :   Tuple				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.eval()
											with torch.no_grad():
													_a  :   Optional[int]				 =							model(**self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase	)	)
											_a  :   Any				 =							outputs.attentions
											self.assertEqual(len(_UpperCAmelCase	)			,_UpperCAmelCase	)

											self.assertListEqual(
											    list(attentions[0].shape[-3:]	)			,[self.model_tester.num_heads[0], window_size_squared, window_size_squared]			,)
											_a  :   Optional[Any]				 =							len(_UpperCAmelCase	)

											# Check attention is always last and order is fine
											_a  :   List[Any]				 =							True
											_a  :   List[str]				 =							True
											_a  :   str				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.eval()
											with torch.no_grad():
													_a  :   Optional[Any]				 =							model(**self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase	)	)

											self.assertEqual(out_len + 1			,len(_UpperCAmelCase	)	)

											_a  :   Tuple				 =							outputs.attentions

											self.assertEqual(len(_UpperCAmelCase	)			,_UpperCAmelCase	)

											self.assertListEqual(
											    list(self_attentions[0].shape[-3:]	)			,[self.model_tester.num_heads[0], window_size_squared, window_size_squared]			,)

@require_vision
@require_torch
@slow
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   Dict				 =							SwinaSRImageProcessor()
									_a  :   List[Any]				 =							SwinaSRForImageSuperResolution.from_pretrained('caidas/swin2SR-classical-sr-x2-64'	).to(_UpperCAmelCase	)

									_a  :   Optional[int]				 =							Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'	)
									_a  :   Optional[int]				 =							processor(images=_UpperCAmelCase			,return_tensors='pt'	).to(_UpperCAmelCase	)

									# forward pass
									with torch.no_grad():
											_a  :   str				 =							model(**_UpperCAmelCase	)

									# verify the logits
									_a  :   int				 =							torch.Size([1, 3, 976, 1296]	)
									self.assertEqual(outputs.reconstruction.shape			,_UpperCAmelCase	)
									_a  :   Any				 =							torch.tensor(
									    [[0.54_58, 0.55_46, 0.56_38], [0.55_26, 0.55_65, 0.56_51], [0.53_96, 0.54_26, 0.56_21]]	).to(_UpperCAmelCase	)
									self.assertTrue(torch.allclose(outputs.reconstruction[0, 0, :3, :3]			,_UpperCAmelCase			,atol=1E-4	)	)









'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : int=13			,_UpperCAmelCase      : List[str]=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : List[Any]=True			,_UpperCAmelCase      : Tuple=True			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=99			,_UpperCAmelCase      : Optional[Any]=[1, 1, 2]			,_UpperCAmelCase      : Optional[Any]=1			,_UpperCAmelCase      : Any=32			,_UpperCAmelCase      : Optional[Any]=4			,_UpperCAmelCase      : Optional[int]=8			,_UpperCAmelCase      : str=37			,_UpperCAmelCase      : Tuple="gelu_new"			,_UpperCAmelCase      : Optional[int]=0.1			,_UpperCAmelCase      : List[Any]=0.1			,_UpperCAmelCase      : Dict=0.0			,_UpperCAmelCase      : str=512			,_UpperCAmelCase      : Any=3			,_UpperCAmelCase      : int=0.02			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : Optional[Any]=None			,_UpperCAmelCase      : Optional[int]=False			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Optional[Any]				 =							batch_size
									_a  :   Union[str, Any]				 =							seq_length
									_a  :   int				 =							is_training
									_a  :   List[Any]				 =							use_input_mask
									_a  :   Dict				 =							use_token_type_ids
									_a  :   List[str]				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   Union[str, Any]				 =							block_sizes
									_a  :   int				 =							num_decoder_layers
									_a  :   List[str]				 =							d_model
									_a  :   Optional[Any]				 =							n_head
									_a  :   int				 =							d_head
									_a  :   Union[str, Any]				 =							d_inner
									_a  :   Any				 =							hidden_act
									_a  :   int				 =							hidden_dropout
									_a  :   str				 =							attention_dropout
									_a  :   Any				 =							activation_dropout
									_a  :   List[Any]				 =							max_position_embeddings
									_a  :   Optional[int]				 =							type_vocab_size
									_a  :   Optional[int]				 =							2
									_a  :   Dict				 =							num_labels
									_a  :   List[str]				 =							num_choices
									_a  :   List[str]				 =							scope
									_a  :   Any				 =							initializer_std

									# Used in the tests to check the size of the first attention layer
									_a  :   List[Any]				 =							n_head
									# Used in the tests to check the size of the first hidden state
									_a  :   List[Any]				 =							self.d_model
									# Used in the tests to check the number of output hidden states/attentions
									_a  :   Tuple				 =							sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
									# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
									# the last hidden state of the first block (which is the first hidden state of the decoder).
									if not base:
											_a  :   Tuple				 =							self.num_hidden_layers + 2







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   Dict				 =							None
									if self.use_input_mask:
											_a  :   str				 =							ids_tensor([self.batch_size, self.seq_length]			,vocab_size=2	)

									_a  :   Any				 =							None
									if self.use_token_type_ids:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   str				 =							None
									_a  :   Optional[int]				 =							None
									_a  :   str				 =							None
									if self.use_labels:
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   List[Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   List[str]				 =							ids_tensor([self.batch_size]			,self.num_choices	)
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,1	)

									_a  :   Dict				 =							self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    fake_token_labels,
									)







							def 							__lowercase    (							self      : Any	):
									return FunnelConfig(
									    vocab_size=self.vocab_size			,block_sizes=self.block_sizes			,num_decoder_layers=self.num_decoder_layers			,d_model=self.d_model			,n_head=self.n_head			,d_head=self.d_head			,d_inner=self.d_inner			,hidden_act=self.hidden_act			,hidden_dropout=self.hidden_dropout			,attention_dropout=self.attention_dropout			,activation_dropout=self.activation_dropout			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_std=self.initializer_std			,)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,):
									_a  :   int				 =							FunnelModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   int				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Optional[Any]				 =							False
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Any			,):
									_a  :   int				 =							FunnelBaseModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Tuple				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)

									_a  :   Dict				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 3, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   List[Any]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,):
									_a  :   Union[str, Any]				 =							self.num_labels
									_a  :   Any				 =							FunnelForPreTraining(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,):
									_a  :   Tuple				 =							FunnelForMaskedLM(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   List[str]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   List[str]				 =							self.num_labels
									_a  :   Dict				 =							FunnelForSequenceClassification(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : int			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   int				 =							self.num_choices
									_a  :   Dict				 =							FunnelForMultipleChoice(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							input_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   List[Any]				 =							token_type_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Tuple				 =							input_mask.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase			,)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,):
									_a  :   Any				 =							self.num_labels
									_a  :   int				 =							FunnelForTokenClassification(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[Any]			,):
									_a  :   List[str]				 =							FunnelForQuestionAnswering(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,start_positions=_UpperCAmelCase			,end_positions=_UpperCAmelCase			,)
									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Tuple				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   List[str]				 =							config_and_inputs
									_a  :   int				 =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					Optional[Any]            =      (
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[int]=False	):
									_a  :   List[Any]				 =							super()._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)

									if return_labels:
											if model_class in get_values(_UpperCAmelCase	):
													_a  :   Optional[int]				 =							torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=_UpperCAmelCase	)
									return inputs_dict







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   int				 =							FunnelModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Union[str, Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   str				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Optional[int]            =      False
							lowerCAmelCase :					Optional[Any]            =      False
							lowerCAmelCase :					Optional[Any]            =      (
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							FunnelModelTester(self			,base=_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_base_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a ,      _a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   str				 =							True

									for model_class in self.all_model_classes:
											if model_class.__name__ == "FunnelBaseModel":
													continue
											_a  :   Optional[int]				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.train()
											_a  :   Dict				 =							self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)
											_a  :   Dict				 =							model(**_UpperCAmelCase	).loss
											loss.backward()







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   Tuple				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Any	):
									_a  :   List[Any]				 =							13
									_a  :   Dict				 =							7
									_a  :   Union[str, Any]				 =							torch.arange(0			,batch_size * sequence_length	).long().reshape(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   Tuple				 =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
									_a  :   List[Any]				 =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

									_a  :   Tuple				 =							FunnelModel.from_pretrained('sgugger/funnel-random-tiny'	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Optional[int]				 =							torch.tensor(23_44.83_52	)
									_a  :   Union[str, Any]				 =							torch.tensor(0.80_52	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)

									_a  :   Tuple				 =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Union[str, Any]				 =							torch.tensor(23_43.84_25	)
									_a  :   Tuple				 =							torch.tensor(0.80_49	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Union[str, Any]				 =							FunnelTokenizer.from_pretrained('huggingface/funnel-small'	)
									_a  :   Tuple				 =							FunnelModel.from_pretrained('huggingface/funnel-small'	)
									_a  :   Union[str, Any]				 =							tokenizer('Hello! I am the Funnel Transformer model.'			,return_tensors='pt'	)
									_a  :   str				 =							model(**_UpperCAmelCase	)[0]

									_a  :   str				 =							torch.tensor(2_35.72_46	)
									_a  :   List[str]				 =							torch.tensor(0.02_56	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)




