



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          RemBertConfig,
          RemBertForCausalLM,
          RemBertForMaskedLM,
          RemBertForMultipleChoice,
          RemBertForQuestionAnswering,
          RemBertForSequenceClassification,
          RemBertForTokenClassification,
          RemBertModel,
      )
      from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class 						snake_case_ :






   def __init__(   self       :  Union[str, Any]			,		lowercase_       :  Any			,		lowercase_       :  str=13			,		lowercase_       :  List[Any]=7			,		lowercase_       :  Optional[Any]=True			,		lowercase_       :  int=True			,		lowercase_       :  List[Any]=True			,		lowercase_       :  Dict=True			,		lowercase_       :  Any=99			,		lowercase_       :  Union[str, Any]=32			,		lowercase_       :  Tuple=18			,		lowercase_       :  List[Any]=43			,		lowercase_       :  Union[str, Any]=5			,		lowercase_       :  str=4			,		lowercase_       :  List[Any]=37			,		lowercase_       :  Dict="gelu"			,		lowercase_       :  int=0.1			,		lowercase_       :  Optional[int]=0.1			,		lowercase_       :  Optional[Any]=5_12			,		lowercase_       :  Optional[int]=16			,		lowercase_       :  List[str]=2			,		lowercase_       :  List[Any]=0.02			,		lowercase_       :  Tuple=3			,		lowercase_       :  Union[str, Any]=4			,		lowercase_       :  str=None			,		)      ->		str:
       lowercase__				:     Tuple					      =       parent
       lowercase__				:     List[Any]					      =       batch_size
       lowercase__				:     List[str]					      =       seq_length
       lowercase__				:     List[Any]					      =       is_training
       lowercase__				:     Optional[Any]					      =       use_input_mask
       lowercase__				:     str					      =       use_token_type_ids
       lowercase__				:     Any					      =       use_labels
       lowercase__				:     Union[str, Any]					      =       vocab_size
       lowercase__				:     Optional[int]					      =       hidden_size
       lowercase__				:     int					      =       input_embedding_size
       lowercase__				:     List[Any]					      =       output_embedding_size
       lowercase__				:     str					      =       num_hidden_layers
       lowercase__				:     int					      =       num_attention_heads
       lowercase__				:     Dict					      =       intermediate_size
       lowercase__				:     str					      =       hidden_act
       lowercase__				:     str					      =       hidden_dropout_prob
       lowercase__				:     Optional[Any]					      =       attention_probs_dropout_prob
       lowercase__				:     Any					      =       max_position_embeddings
       lowercase__				:     Dict					      =       type_vocab_size
       lowercase__				:     str					      =       type_sequence_label_size
       lowercase__				:     Dict					      =       initializer_range
       lowercase__				:     int					      =       num_labels
       lowercase__				:     int					      =       num_choices
       lowercase__				:     Dict					      =       scope






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		List[Any]:
       lowercase__				:     List[Any]					      =       ids_tensor([self.batch_size, self.seq_length]			,		self.vocab_size						)

       lowercase__				:     List[Any]					      =       None
       if self.use_input_mask:
           lowercase__				:     Optional[Any]					      =       random_attention_mask([self.batch_size, self.seq_length]						)

       lowercase__				:     List[Any]					      =       None
       if self.use_token_type_ids:
           lowercase__				:     List[Any]					      =       ids_tensor([self.batch_size, self.seq_length]			,		self.type_vocab_size						)

       lowercase__				:     Optional[Any]					      =       None
       lowercase__				:     Optional[int]					      =       None
       lowercase__				:     Dict					      =       None
       if self.use_labels:
           lowercase__				:     str					      =       ids_tensor([self.batch_size]			,		self.type_sequence_label_size						)
           lowercase__				:     Optional[int]					      =       ids_tensor([self.batch_size, self.seq_length]			,		self.num_labels						)
           lowercase__				:     Dict					      =       ids_tensor([self.batch_size]			,		self.num_choices						)

       lowercase__				:     List[str]					      =       RemBertConfig(
           vocab_size=self.vocab_size			,		hidden_size=self.hidden_size			,		input_embedding_size=self.input_embedding_size			,		output_embedding_size=self.output_embedding_size			,		num_hidden_layers=self.num_hidden_layers			,		num_attention_heads=self.num_attention_heads			,		intermediate_size=self.intermediate_size			,		hidden_act=self.hidden_act			,		hidden_dropout_prob=self.hidden_dropout_prob			,		attention_probs_dropout_prob=self.attention_probs_dropout_prob			,		max_position_embeddings=self.max_position_embeddings			,		type_vocab_size=self.type_vocab_size			,		is_decoder=__A			,		initializer_range=self.initializer_range			,		)

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






   def    __UpperCamelCase  (   self       :  List[Any]						)      ->		Tuple:
       (
           lowercase__					
       )				:     Optional[Any]					      =       self.prepare_config_and_inputs()

       lowercase__				:     Optional[Any]					      =       True
       lowercase__				:     Dict					      =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
       lowercase__				:     Optional[int]					      =       ids_tensor([self.batch_size, self.seq_length]			,		vocab_size=2						)

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )






   def    __UpperCamelCase  (   self       :  Optional[Any]			,		lowercase_       :  List[str]			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  List[str]			,		lowercase_       :  Dict			,		lowercase_       :  Dict			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  List[Any]						)      ->		Dict:
       lowercase__				:     List[str]					      =       RemBertModel(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     Tuple					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A						)
       lowercase__				:     Dict					      =       model(__A			,		token_type_ids=__A						)
       lowercase__				:     int					      =       model(__A						)
       self.parent.assertEqual(result.last_hidden_state.shape			,		(self.batch_size, self.seq_length, self.hidden_size)						)






   def    __UpperCamelCase  (   self       :  Optional[Any]			,		lowercase_       :  str			,		lowercase_       :  List[Any]			,		lowercase_       :  Dict			,		lowercase_       :  Optional[int]			,		lowercase_       :  Any			,		lowercase_       :  Optional[Any]			,		lowercase_       :  Optional[Any]			,		lowercase_       :  List[str]			,		lowercase_       :  List[str]			,		)      ->		int:
       lowercase__				:     Union[str, Any]					      =       True
       lowercase__				:     Union[str, Any]					      =       RemBertModel(__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     List[str]					      =       model(
           __A			,		attention_mask=__A			,		token_type_ids=__A			,		encoder_hidden_states=__A			,		encoder_attention_mask=__A			,		)
       lowercase__				:     str					      =       model(
           __A			,		attention_mask=__A			,		token_type_ids=__A			,		encoder_hidden_states=__A			,		)
       lowercase__				:     List[Any]					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A						)
       self.parent.assertEqual(result.last_hidden_state.shape			,		(self.batch_size, self.seq_length, self.hidden_size)						)






   def    __UpperCamelCase  (   self       :  Optional[Any]			,		lowercase_       :  Optional[int]			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Any			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  int			,		lowercase_       :  List[Any]			,		lowercase_       :  str			,		lowercase_       :  Dict			,		lowercase_       :  Optional[int]			,		)      ->		Tuple:
       lowercase__				:     Dict					      =       RemBertForCausalLM(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     int					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A			,		labels=__A						)
       self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.vocab_size)						)






   def    __UpperCamelCase  (   self       :  Union[str, Any]			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Any			,		lowercase_       :  Optional[int]			,		lowercase_       :  Any			,		lowercase_       :  Tuple			,		lowercase_       :  int			,		lowercase_       :  Tuple						)      ->		Any:
       lowercase__				:     int					      =       RemBertForMaskedLM(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     Optional[int]					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A			,		labels=__A						)
       self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.vocab_size)						)






   def    __UpperCamelCase  (   self       :  Optional[int]			,		lowercase_       :  str			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Dict			,		lowercase_       :  str			,		lowercase_       :  Optional[int]			,		lowercase_       :  Tuple			,		lowercase_       :  Optional[Any]			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Dict			,		)      ->		str:
       lowercase__				:     Optional[Any]					      =       True
       lowercase__				:     Tuple					      =       True
       lowercase__				:     str					      =       RemBertForCausalLM(config=__A						)
       model.to(__A						)
       model.eval()

       # first forward pass
       lowercase__				:     Optional[Any]					      =       model(
           __A			,		attention_mask=__A			,		encoder_hidden_states=__A			,		encoder_attention_mask=__A			,		use_cache=__A			,		)
       lowercase__				:     List[str]					      =       outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       lowercase__				:     Tuple					      =       ids_tensor((self.batch_size, 3)			,		config.vocab_size						)
       lowercase__				:     Any					      =       ids_tensor((self.batch_size, 3)			,		vocab_size=2						)

       # append to next input_ids and
       lowercase__				:     Tuple					      =       torch.cat([input_ids, next_tokens]			,		dim=-1						)
       lowercase__				:     Any					      =       torch.cat([input_mask, next_mask]			,		dim=-1						)

       lowercase__				:     Dict					      =       model(
           __A			,		attention_mask=__A			,		encoder_hidden_states=__A			,		encoder_attention_mask=__A			,		output_hidden_states=__A			,		)["hidden_states"][0]
       lowercase__				:     Dict					      =       model(
           __A			,		attention_mask=__A			,		encoder_hidden_states=__A			,		encoder_attention_mask=__A			,		past_key_values=__A			,		output_hidden_states=__A			,		)["hidden_states"][0]

       # select random slice
       lowercase__				:     int					      =       ids_tensor((1,)			,		output_from_past.shape[-1]						).item()
       lowercase__				:     str					      =       output_from_no_past[:, -3:, random_slice_idx].detach()
       lowercase__				:     Tuple					      =       output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(__A			,		__A			,		atol=1E-3						)						)






   def    __UpperCamelCase  (   self       :  List[str]			,		lowercase_       :  Optional[Any]			,		lowercase_       :  List[Any]			,		lowercase_       :  str			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  List[str]			,		lowercase_       :  Tuple			,		lowercase_       :  Optional[Any]						)      ->		List[str]:
       lowercase__				:     int					      =       RemBertForQuestionAnswering(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     Optional[Any]					      =       model(
           __A			,		attention_mask=__A			,		token_type_ids=__A			,		start_positions=__A			,		end_positions=__A			,		)
       self.parent.assertEqual(result.start_logits.shape			,		(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape			,		(self.batch_size, self.seq_length)						)






   def    __UpperCamelCase  (   self       :  Optional[Any]			,		lowercase_       :  int			,		lowercase_       :  Optional[Any]			,		lowercase_       :  List[Any]			,		lowercase_       :  Any			,		lowercase_       :  int			,		lowercase_       :  Optional[Any]			,		lowercase_       :  Dict						)      ->		Optional[int]:
       lowercase__				:     int					      =       self.num_labels
       lowercase__				:     Dict					      =       RemBertForSequenceClassification(__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     List[str]					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A			,		labels=__A						)
       self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.num_labels)						)






   def    __UpperCamelCase  (   self       :  Optional[int]			,		lowercase_       :  int			,		lowercase_       :  List[str]			,		lowercase_       :  Any			,		lowercase_       :  int			,		lowercase_       :  int			,		lowercase_       :  Dict			,		lowercase_       :  List[Any]						)      ->		List[Any]:
       lowercase__				:     Optional[int]					      =       self.num_labels
       lowercase__				:     Optional[int]					      =       RemBertForTokenClassification(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     List[str]					      =       model(__A			,		attention_mask=__A			,		token_type_ids=__A			,		labels=__A						)
       self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.num_labels)						)






   def    __UpperCamelCase  (   self       :  int			,		lowercase_       :  Optional[Any]			,		lowercase_       :  List[Any]			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Dict			,		lowercase_       :  Optional[int]			,		lowercase_       :  Optional[int]			,		lowercase_       :  Optional[int]						)      ->		Tuple:
       lowercase__				:     Optional[Any]					      =       self.num_choices
       lowercase__				:     Optional[int]					      =       RemBertForMultipleChoice(config=__A						)
       model.to(__A						)
       model.eval()
       lowercase__				:     Optional[Any]					      =       input_ids.unsqueeze(1						).expand(-1			,		self.num_choices			,		-1						).contiguous()
       lowercase__				:     str					      =       token_type_ids.unsqueeze(1						).expand(-1			,		self.num_choices			,		-1						).contiguous()
       lowercase__				:     Optional[Any]					      =       input_mask.unsqueeze(1						).expand(-1			,		self.num_choices			,		-1						).contiguous()
       lowercase__				:     int					      =       model(
           __A			,		attention_mask=__A			,		token_type_ids=__A			,		labels=__A			,		)
       self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.num_choices)						)







   def    __UpperCamelCase  (   self       :  Dict						)      ->		Optional[int]:
       lowercase__				:     List[Any]					      =       self.prepare_config_and_inputs()
       (
           lowercase__					
       )				:     Optional[int]					      =       config_and_inputs
       lowercase__				:     str					      =       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict



@require_torch
class 						snake_case_ (	a__						,a__						,unittest.TestCase ):
   __A			:	Optional[int]        =       (
       (
           RemBertModel,
           RemBertForMaskedLM,
           RemBertForCausalLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __A			:	Tuple        =       (RemBertForCausalLM,) if is_torch_available() else ()
   __A			:	Optional[int]        =       (
       {
           "feature-extraction": RemBertModel,
           "fill-mask": RemBertForMaskedLM,
           "question-answering": RemBertForQuestionAnswering,
           "text-classification": RemBertForSequenceClassification,
           "text-generation": RemBertForCausalLM,
           "token-classification": RemBertForTokenClassification,
           "zero-shot": RemBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )






   def    __UpperCamelCase  (   self       :  Optional[int]						)      ->		Tuple:
       lowercase__				:     Tuple					      =       RemBertModelTester(self						)
       lowercase__				:     Optional[Any]					      =       ConfigTester(self			,		config_class=__A			,		hidden_size=37						)






   def    __UpperCamelCase  (   self       :  Dict						)      ->		Dict:
       self.config_tester.run_common_tests()






   def    __UpperCamelCase  (   self       :  str						)      ->		Union[str, Any]:
       lowercase__				:     List[Any]					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__A						)






   def    __UpperCamelCase  (   self       :  str						)      ->		Any:
       lowercase__				:     Tuple					      =       self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
           lowercase__				:     int					      =       type
           self.model_tester.create_and_check_model(*__A						)






   def    __UpperCamelCase  (   self       :  Tuple						)      ->		Dict:
       lowercase__				:     Optional[int]					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*__A						)






   def    __UpperCamelCase  (   self       :  Tuple						)      ->		Optional[Any]:
       lowercase__				:     Tuple					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*__A						)






   def    __UpperCamelCase  (   self       :  Tuple						)      ->		List[str]:
       lowercase__				:     Tuple					      =       self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(*__A						)






   def    __UpperCamelCase  (   self       :  int						)      ->		Any:
       lowercase__				:     Optional[int]					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__A						)






   def    __UpperCamelCase  (   self       :  str						)      ->		Any:
       lowercase__				:     Dict					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__A						)






   def    __UpperCamelCase  (   self       :  int						)      ->		Any:
       lowercase__				:     Tuple					      =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__A						)






   def    __UpperCamelCase  (   self       :  Optional[Any]						)      ->		List[str]:
       lowercase__				:     Any					      =       self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(*__A						)






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		int:
       (
           lowercase__					
       )				:     Any					      =       self.model_tester.prepare_config_and_inputs_for_decoder()

       lowercase__				:     Union[str, Any]					      =       None

       self.model_tester.create_and_check_model_as_decoder(
           __A			,		__A			,		__A			,		__A			,		__A			,		__A			,		__A			,		__A			,		__A			,		)







   @slow
   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Union[str, Any]:







       for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase__				:     Any					      =       RemBertModel.from_pretrained(__A						)
           self.assertIsNotNone(__A						)



@require_torch
class 						snake_case_ (	unittest.TestCase ):






   @slow
   def    __UpperCamelCase  (   self       :  int						)      ->		List[str]:
       lowercase__				:     List[Any]					      =       RemBertModel.from_pretrained("google/rembert"						)
       lowercase__				:     Tuple					      =       torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]						)
       lowercase__				:     str					      =       torch.tensor([[0, 0, 0, 1, 1]]						)
       with torch.no_grad():
           lowercase__				:     Optional[Any]					      =       model(__A			,		token_type_ids=__A			,		output_hidden_states=__A						)

       lowercase__				:     int					      =       11_52

       lowercase__				:     Any					      =       torch.Size((1, 5, hidden_size)						)
       self.assertEqual(output["last_hidden_state"].shape			,		__A						)

       lowercase__				:     Optional[Any]					      =       torch.tensor(
           [
               [
                   [0.07_54, -0.20_22, 0.19_04],
                   [-0.33_54, -0.36_92, -0.47_91],
                   [-0.23_14, -0.67_29, -0.07_49],
                   [-0.03_96, -0.31_05, -0.42_34],
                   [-0.15_71, -0.05_25, 0.53_53],
               ]
           ]						)

       # Running on the original tf implementation gives slightly different results here.
       # Not clear why this variations is present
       # TODO: Find reason for discrepancy
       # expected_original_implementation = [[
       #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
       #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
       #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
       #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
       #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
       # ]]

       self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3]			,		__A			,		atol=1E-4						)						)




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCamelCase													=		{
    '''configuration_mobilenet_v1''': [
        '''MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''MobileNetV1Config''',
        '''MobileNetV1OnnxConfig''',
    ],
}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase													=		['''MobileNetV1FeatureExtractor''']
      UpperCamelCase													=		['''MobileNetV1ImageProcessor''']

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase													=		[
          '''MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''MobileNetV1ForImageClassification''',
          '''MobileNetV1Model''',
          '''MobileNetV1PreTrainedModel''',
          '''load_tf_weights_in_mobilenet_v1''',
      ]


if TYPE_CHECKING:
      from .configuration_mobilenet_va import (
          MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP,
          MobileNetVaConfig,
          MobileNetVaOnnxConfig,
      )

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_mobilenet_va import MobileNetVaFeatureExtractor
            from .image_processing_mobilenet_va import MobileNetVaImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mobilenet_va import (
                MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST,
                MobileNetVaForImageClassification,
                MobileNetVaModel,
                MobileNetVaPreTrainedModel,
                load_tf_weights_in_mobilenet_va,
            )


else:
      import sys

      UpperCamelCase													=		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



