





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    RemBertConfig,
	    RemBertForCausalLM,
	    RemBertForMaskedLM,
	    RemBertForMultipleChoice,
	    RemBertForQuestionAnswering,
	    RemBertForSequenceClassification,
	    RemBertForTokenClassification,
	    RemBertModel,
	)
	from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class     lowerCamelCase:
					'''simple docstring'''


					def __init__(						self							,    snake_case_							,    snake_case_=13							,    snake_case_=7							,    snake_case_=True							,    snake_case_=True							,    snake_case_=True							,    snake_case_=True							,    snake_case_=99							,    snake_case_=32							,    snake_case_=18							,    snake_case_=43							,    snake_case_=5							,    snake_case_=4							,    snake_case_=37							,    snake_case_="gelu"							,    snake_case_=0.1							,    snake_case_=0.1							,    snake_case_=512							,    snake_case_=16							,    snake_case_=2							,    snake_case_=0.02							,    snake_case_=3							,    snake_case_=4							,    snake_case_=None							,    ):
									_A    =       parent
									_A    =       batch_size
									_A    =       seq_length
									_A    =       is_training
									_A    =       use_input_mask
									_A    =       use_token_type_ids
									_A    =       use_labels
									_A    =       vocab_size
									_A    =       hidden_size
									_A    =       input_embedding_size
									_A    =       output_embedding_size
									_A    =       num_hidden_layers
									_A    =       num_attention_heads
									_A    =       intermediate_size
									_A    =       hidden_act
									_A    =       hidden_dropout_prob
									_A    =       attention_probs_dropout_prob
									_A    =       max_position_embeddings
									_A    =       type_vocab_size
									_A    =       type_sequence_label_size
									_A    =       initializer_range
									_A    =       num_labels
									_A    =       num_choices
									_A    =       scope




					def 					lowerCAmelCase__		(						self       ):
									_A    =       ids_tensor([self.batch_size, self.seq_length]							,    self.vocab_size       )

									_A    =       None
									if self.use_input_mask:
													_A    =       random_attention_mask([self.batch_size, self.seq_length]       )

									_A    =       None
									if self.use_token_type_ids:
													_A    =       ids_tensor([self.batch_size, self.seq_length]							,    self.type_vocab_size       )

									_A    =       None
									_A    =       None
									_A    =       None
									if self.use_labels:
													_A    =       ids_tensor([self.batch_size]							,    self.type_sequence_label_size       )
													_A    =       ids_tensor([self.batch_size, self.seq_length]							,    self.num_labels       )
													_A    =       ids_tensor([self.batch_size]							,    self.num_choices       )

									_A    =       RemBertConfig(
									    vocab_size=self.vocab_size							,    hidden_size=self.hidden_size							,    input_embedding_size=self.input_embedding_size							,    output_embedding_size=self.output_embedding_size							,    num_hidden_layers=self.num_hidden_layers							,    num_attention_heads=self.num_attention_heads							,    intermediate_size=self.intermediate_size							,    hidden_act=self.hidden_act							,    hidden_dropout_prob=self.hidden_dropout_prob							,    attention_probs_dropout_prob=self.attention_probs_dropout_prob							,    max_position_embeddings=self.max_position_embeddings							,    type_vocab_size=self.type_vocab_size							,    is_decoder=snake_case_							,    initializer_range=self.initializer_range							,    )

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




					def 					lowerCAmelCase__		(						self       ):
									(
									    (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									),        
									)    =       self.prepare_config_and_inputs()

									_A    =       True
									_A    =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
									_A    =       ids_tensor([self.batch_size, self.seq_length]							,    vocab_size=2       )

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    encoder_hidden_states,
									    encoder_attention_mask,
									)




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       RemBertModel(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_       )
									_A    =       model(snake_case_							,    token_type_ids=snake_case_       )
									_A    =       model(snake_case_       )
									self.parent.assertEqual(result.last_hidden_state.shape							,    (self.batch_size, self.seq_length, self.hidden_size)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    ):
									_A    =       True
									_A    =       RemBertModel(snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    encoder_hidden_states=snake_case_							,    encoder_attention_mask=snake_case_							,    )
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    encoder_hidden_states=snake_case_							,    )
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_       )
									self.parent.assertEqual(result.last_hidden_state.shape							,    (self.batch_size, self.seq_length, self.hidden_size)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    ):
									_A    =       RemBertForCausalLM(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    labels=snake_case_       )
									self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.vocab_size)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       RemBertForMaskedLM(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    labels=snake_case_       )
									self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.vocab_size)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    ):
									_A    =       True
									_A    =       True
									_A    =       RemBertForCausalLM(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()

									# first forward pass
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    encoder_hidden_states=snake_case_							,    encoder_attention_mask=snake_case_							,    use_cache=snake_case_							,    )
									_A    =       outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									_A    =       ids_tensor((self.batch_size, 3)							,    config.vocab_size       )
									_A    =       ids_tensor((self.batch_size, 3)							,    vocab_size=2       )

									# append to next input_ids and
									_A    =       torch.cat([input_ids, next_tokens]							,    dim=-1       )
									_A    =       torch.cat([input_mask, next_mask]							,    dim=-1       )

									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    encoder_hidden_states=snake_case_							,    encoder_attention_mask=snake_case_							,    output_hidden_states=snake_case_							,    )['hidden_states'][0]
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    encoder_hidden_states=snake_case_							,    encoder_attention_mask=snake_case_							,    past_key_values=snake_case_							,    output_hidden_states=snake_case_							,    )['hidden_states'][0]

									# select random slice
									_A    =       ids_tensor((1,)							,    output_from_past.shape[-1]       ).item()
									_A    =       output_from_no_past[:, -3:, random_slice_idx].detach()
									_A    =       output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(snake_case_							,    snake_case_							,    atol=1E-3       )       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       RemBertForQuestionAnswering(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    start_positions=snake_case_							,    end_positions=snake_case_							,    )
									self.parent.assertEqual(result.start_logits.shape							,    (self.batch_size, self.seq_length)       )
									self.parent.assertEqual(result.end_logits.shape							,    (self.batch_size, self.seq_length)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       self.num_labels
									_A    =       RemBertForSequenceClassification(snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    labels=snake_case_       )
									self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_labels)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       self.num_labels
									_A    =       RemBertForTokenClassification(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       model(snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    labels=snake_case_       )
									self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.num_labels)       )




					def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_       ):
									_A    =       self.num_choices
									_A    =       RemBertForMultipleChoice(config=snake_case_       )
									model.to(snake_case_       )
									model.eval()
									_A    =       input_ids.unsqueeze(1       ).expand(-1							,    self.num_choices							,    -1       ).contiguous()
									_A    =       token_type_ids.unsqueeze(1       ).expand(-1							,    self.num_choices							,    -1       ).contiguous()
									_A    =       input_mask.unsqueeze(1       ).expand(-1							,    self.num_choices							,    -1       ).contiguous()
									_A    =       model(
									    snake_case_							,    attention_mask=snake_case_							,    token_type_ids=snake_case_							,    labels=snake_case_							,    )
									self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_choices)       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.prepare_config_and_inputs()
									(
									    (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									),        
									)    =       config_and_inputs
									_A    =       {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict







@require_torch
class     lowerCamelCase(	__lowerCamelCase ,   __lowerCamelCase ,   unittest.TestCase  ):
					'''simple docstring'''


					__magic_name__        = (
					    (
					        RemBertModel,
					        RemBertForMaskedLM,
					        RemBertForCausalLM,
					        RemBertForMultipleChoice,
					        RemBertForQuestionAnswering,
					        RemBertForSequenceClassification,
					        RemBertForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					__magic_name__        = (RemBertForCausalLM,) if is_torch_available() else ()
					__magic_name__        = (
					    {
					        'feature-extraction': RemBertModel,
					        'fill-mask': RemBertForMaskedLM,
					        'question-answering': RemBertForQuestionAnswering,
					        'text-classification': RemBertForSequenceClassification,
					        'text-generation': RemBertForCausalLM,
					        'token-classification': RemBertForTokenClassification,
					        'zero-shot': RemBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)




					def 					lowerCAmelCase__		(						self       ):
									_A    =       RemBertModelTester(self       )
									_A    =       ConfigTester(self							,    config_class=snake_case_							,    hidden_size=37       )




					def 					lowerCAmelCase__		(						self       ):
									self.config_tester.run_common_tests()




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
													_A    =       type
													self.model_tester.create_and_check_model(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_decoder_model_past_large_inputs(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									_A    =       self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_model_as_decoder(*snake_case_       )




					def 					lowerCAmelCase__		(						self       ):
									(
									    (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									), (
									    _A       
									),        
									)    =       self.model_tester.prepare_config_and_inputs_for_decoder()

									_A    =       None

									self.model_tester.create_and_check_model_as_decoder(
									    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    )




					@slow
					def 					lowerCAmelCase__		(						self       ):






									for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													_A    =       RemBertModel.from_pretrained(snake_case_       )
													self.assertIsNotNone(snake_case_       )







@require_torch
class     lowerCamelCase(	unittest.TestCase  ):
					'''simple docstring'''


					@slow
					def 					lowerCAmelCase__		(						self       ):
									_A    =       RemBertModel.from_pretrained('google/rembert'       )
									_A    =       torch.tensor([[312, 5_6498, 313, 2125, 313]]       )
									_A    =       torch.tensor([[0, 0, 0, 1, 1]]       )
									with torch.no_grad():
													_A    =       model(snake_case_							,    token_type_ids=snake_case_							,    output_hidden_states=snake_case_       )

									_A    =       1152

									_A    =       torch.Size((1, 5, hidden_size)       )
									self.assertEqual(output['last_hidden_state'].shape							,    snake_case_       )

									_A    =       torch.tensor(
									    [
									        [
									            [0.0754, -0.2022, 0.1904],
									            [-0.3354, -0.3692, -0.4791],
									            [-0.2314, -0.6729, -0.0749],
									            [-0.0396, -0.3105, -0.4234],
									            [-0.1571, -0.0525, 0.5353],
									        ]
									    ]       )

									# Running on the original tf implementation gives slightly different results here.
									# Not clear why this variations is present
									# TODO: Find reason for discrepancy
									# expected_original_implementation = [[
									#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
									#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
									#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
									#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
									#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
									# ]]

									self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]							,    snake_case_							,    atol=1E-4       )       )












from ..utils import DummyObject, requires_backends


__A       : str						    = None





def 							__lowerCAmelCase(					*_SCREAMING_SNAKE_CASE       ,    **_SCREAMING_SNAKE_CASE							)       ->   Tuple:




    """simple docstring"""




    requires_backends(_SCREAMING_SNAKE_CASE       ,    ['sentencepiece', 'tokenizers']							)





