



"""simple docstring"""





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RemBertConfig,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertModel,
  )
  from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class       __lowerCamelCase	:

   '''simple docstring'''





   def __init__(self       ,	lowerCamelCase       ,	lowerCamelCase=13       ,	lowerCamelCase=7       ,	lowerCamelCase=True       ,	lowerCamelCase=True       ,	lowerCamelCase=True       ,	lowerCamelCase=True       ,	lowerCamelCase=99       ,	lowerCamelCase=32       ,	lowerCamelCase=18       ,	lowerCamelCase=43       ,	lowerCamelCase=5       ,	lowerCamelCase=4       ,	lowerCamelCase=37       ,	lowerCamelCase="gelu"       ,	lowerCamelCase=0.1       ,	lowerCamelCase=0.1       ,	lowerCamelCase=512       ,	lowerCamelCase=16       ,	lowerCamelCase=2       ,	lowerCamelCase=0.02       ,	lowerCamelCase=3       ,	lowerCamelCase=4       ,	lowerCamelCase=None       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		parent
      _lowerCAmelCase					     =		batch_size
      _lowerCAmelCase					     =		seq_length
      _lowerCAmelCase					     =		is_training
      _lowerCAmelCase					     =		use_input_mask
      _lowerCAmelCase					     =		use_token_type_ids
      _lowerCAmelCase					     =		use_labels
      _lowerCAmelCase					     =		vocab_size
      _lowerCAmelCase					     =		hidden_size
      _lowerCAmelCase					     =		input_embedding_size
      _lowerCAmelCase					     =		output_embedding_size
      _lowerCAmelCase					     =		num_hidden_layers
      _lowerCAmelCase					     =		num_attention_heads
      _lowerCAmelCase					     =		intermediate_size
      _lowerCAmelCase					     =		hidden_act
      _lowerCAmelCase					     =		hidden_dropout_prob
      _lowerCAmelCase					     =		attention_probs_dropout_prob
      _lowerCAmelCase					     =		max_position_embeddings
      _lowerCAmelCase					     =		type_vocab_size
      _lowerCAmelCase					     =		type_sequence_label_size
      _lowerCAmelCase					     =		initializer_range
      _lowerCAmelCase					     =		num_labels
      _lowerCAmelCase					     =		num_choices
      _lowerCAmelCase					     =		scope







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size				)

      _lowerCAmelCase					     =		None
      if self.use_input_mask:
         _lowerCAmelCase					     =		random_attention_mask([self.batch_size, self.seq_length]				)

      _lowerCAmelCase					     =		None
      if self.use_token_type_ids:
         _lowerCAmelCase					     =		ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size				)

      _lowerCAmelCase					     =		None
      _lowerCAmelCase					     =		None
      _lowerCAmelCase					     =		None
      if self.use_labels:
         _lowerCAmelCase					     =		ids_tensor([self.batch_size]       ,	self.type_sequence_label_size				)
         _lowerCAmelCase					     =		ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels				)
         _lowerCAmelCase					     =		ids_tensor([self.batch_size]       ,	self.num_choices				)

      _lowerCAmelCase					     =		RemBertConfig(
          vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	input_embedding_size=self.input_embedding_size       ,	output_embedding_size=self.output_embedding_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	is_decoder=__SCREAMING_SNAKE_CASE       ,	initializer_range=self.initializer_range       ,	)

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







   def 							A__						(self				):




      '''simple docstring'''




      (
          (
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						   
      )					     =		self.prepare_config_and_inputs()

      _lowerCAmelCase					     =		True
      _lowerCAmelCase					     =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
      _lowerCAmelCase					     =		ids_tensor([self.batch_size, self.seq_length]       ,	vocab_size=2				)

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertModel(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE				)
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE				)
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		True
      _lowerCAmelCase					     =		RemBertModel(__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	encoder_hidden_states=__SCREAMING_SNAKE_CASE       ,	encoder_attention_mask=__SCREAMING_SNAKE_CASE       ,	)
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	encoder_hidden_states=__SCREAMING_SNAKE_CASE       ,	)
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertForCausalLM(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	labels=__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertForMaskedLM(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	labels=__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		True
      _lowerCAmelCase					     =		True
      _lowerCAmelCase					     =		RemBertForCausalLM(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()

      # first forward pass
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	encoder_hidden_states=__SCREAMING_SNAKE_CASE       ,	encoder_attention_mask=__SCREAMING_SNAKE_CASE       ,	use_cache=__SCREAMING_SNAKE_CASE       ,	)
      _lowerCAmelCase					     =		outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      _lowerCAmelCase					     =		ids_tensor((self.batch_size, 3)       ,	config.vocab_size				)
      _lowerCAmelCase					     =		ids_tensor((self.batch_size, 3)       ,	vocab_size=2				)

      # append to next input_ids and
      _lowerCAmelCase					     =		torch.cat([input_ids, next_tokens]       ,	dim=-1				)
      _lowerCAmelCase					     =		torch.cat([input_mask, next_mask]       ,	dim=-1				)

      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	encoder_hidden_states=__SCREAMING_SNAKE_CASE       ,	encoder_attention_mask=__SCREAMING_SNAKE_CASE       ,	output_hidden_states=__SCREAMING_SNAKE_CASE       ,	)["""hidden_states"""][0]
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	encoder_hidden_states=__SCREAMING_SNAKE_CASE       ,	encoder_attention_mask=__SCREAMING_SNAKE_CASE       ,	past_key_values=__SCREAMING_SNAKE_CASE       ,	output_hidden_states=__SCREAMING_SNAKE_CASE       ,	)["""hidden_states"""][0]

      # select random slice
      _lowerCAmelCase					     =		ids_tensor((1,)       ,	output_from_past.shape[-1]				).item()
      _lowerCAmelCase					     =		output_from_no_past[:, -3:, random_slice_idx].detach()
      _lowerCAmelCase					     =		output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	atol=1e-3				)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertForQuestionAnswering(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	start_positions=__SCREAMING_SNAKE_CASE       ,	end_positions=__SCREAMING_SNAKE_CASE       ,	)
      self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)				)
      self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.num_labels
      _lowerCAmelCase					     =		RemBertForSequenceClassification(__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	labels=__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.num_labels
      _lowerCAmelCase					     =		RemBertForTokenClassification(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	labels=__SCREAMING_SNAKE_CASE				)
      self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)				)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.num_choices
      _lowerCAmelCase					     =		RemBertForMultipleChoice(config=__SCREAMING_SNAKE_CASE				)
      model.to(__SCREAMING_SNAKE_CASE				)
      model.eval()
      _lowerCAmelCase					     =		input_ids.unsqueeze(1				).expand(-1       ,	self.num_choices       ,	-1				).contiguous()
      _lowerCAmelCase					     =		token_type_ids.unsqueeze(1				).expand(-1       ,	self.num_choices       ,	-1				).contiguous()
      _lowerCAmelCase					     =		input_mask.unsqueeze(1				).expand(-1       ,	self.num_choices       ,	-1				).contiguous()
      _lowerCAmelCase					     =		model(
          __SCREAMING_SNAKE_CASE       ,	attention_mask=__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	labels=__SCREAMING_SNAKE_CASE       ,	)
      self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_choices)				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.prepare_config_and_inputs()
      (
          (
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						   
      )					     =		config_and_inputs
      _lowerCAmelCase					     =		{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
      return config, inputs_dict





@require_torch
class       __lowerCamelCase	(   UpperCamelCase_   ,							UpperCamelCase_   ,							unittest.TestCase    ):

   '''simple docstring'''





   __UpperCamelCase									=   (
       (
           RemBertModel,
           RemBertForMaskedLM,
           RemBertForCausalLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __UpperCamelCase									=   (RemBertForCausalLM,) if is_torch_available() else ()
   __UpperCamelCase									=   (
       {
           """feature-extraction""": RemBertModel,
           """fill-mask""": RemBertForMaskedLM,
           """question-answering""": RemBertForQuestionAnswering,
           """text-classification""": RemBertForSequenceClassification,
           """text-generation""": RemBertForCausalLM,
           """token-classification""": RemBertForTokenClassification,
           """zero-shot""": RemBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertModelTester(self				)
      _lowerCAmelCase					     =		ConfigTester(self       ,	config_class=__SCREAMING_SNAKE_CASE       ,	hidden_size=37				)







   def 							A__						(self				):




      '''simple docstring'''




      self.config_tester.run_common_tests()







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
         _lowerCAmelCase					     =		type
         self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_decoder_model_past_large_inputs(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_model_as_decoder(*__SCREAMING_SNAKE_CASE				)







   def 							A__						(self				):




      '''simple docstring'''




      (
          (
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						(
          _lowerCAmelCase   
      )     ,						   
      )					     =		self.model_tester.prepare_config_and_inputs_for_decoder()

      _lowerCAmelCase					     =		None

      self.model_tester.create_and_check_model_as_decoder(
          __SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	__SCREAMING_SNAKE_CASE       ,	)







   @slow
   def 							A__						(self				):




      '''simple docstring'''




      for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowerCAmelCase					     =		RemBertModel.from_pretrained(__SCREAMING_SNAKE_CASE				)
         self.assertIsNotNone(__SCREAMING_SNAKE_CASE				)





@require_torch
class       __lowerCamelCase	(   unittest.TestCase    ):

   '''simple docstring'''





   @slow
   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		RemBertModel.from_pretrained("""google/rembert"""				)
      _lowerCAmelCase					     =		torch.tensor([[312, 56_498, 313, 2_125, 313]]				)
      _lowerCAmelCase					     =		torch.tensor([[0, 0, 0, 1, 1]]				)
      with torch.no_grad():
         _lowerCAmelCase					     =		model(__SCREAMING_SNAKE_CASE       ,	token_type_ids=__SCREAMING_SNAKE_CASE       ,	output_hidden_states=__SCREAMING_SNAKE_CASE				)

      _lowerCAmelCase					     =		1_152

      _lowerCAmelCase					     =		torch.Size((1, 5, hidden_size)				)
      self.assertEqual(output["""last_hidden_state"""].shape       ,	__SCREAMING_SNAKE_CASE				)

      _lowerCAmelCase					     =		torch.tensor(
          [
              [
                  [0.0754, -0.2022, 0.1904],
                  [-0.3354, -0.3692, -0.4791],
                  [-0.2314, -0.6729, -0.0749],
                  [-0.0396, -0.3105, -0.4234],
                  [-0.1571, -0.0525, 0.5353],
              ]
          ]				)

      # Running on the original tf implementation gives slightly different results here.
      # Not clear why this variations is present
      # TODO: Find reason for discrepancy
      # expected_original_implementation = [[
      #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
      #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
      #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
      #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
      #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
      # ]]

      self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3]       ,	__SCREAMING_SNAKE_CASE       ,	atol=1e-4				)				)




"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE				:     Optional[Any]          =	logging.get_logger(__name__)


SCREAMING_SNAKE_CASE				:     int          =	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
SCREAMING_SNAKE_CASE				:     Union[str, Any]          =	{
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
}

SCREAMING_SNAKE_CASE				:     Union[str, Any]          =	{
    '''RUCAIBox/mvp''': 1_0_2_4,
}






@lru_cache()
def 	__UpperCAmelCase					(       )							->	Tuple:






   """simple docstring"""


   _lowerCAmelCase					     =		(
       list(range(ord("""!"""   )   ,	ord("""~"""   ) + 1   )   ) + list(range(ord("""¡"""   )   ,	ord("""¬"""   ) + 1   )   ) + list(range(ord("""®"""   )   ,	ord("""ÿ"""   ) + 1   )   )
   )
   _lowerCAmelCase					     =		bs[:]
   _lowerCAmelCase					     =		0
   for b in range(2**8   ):
      if b not in bs:
         bs.append(snake_case_   )
         cs.append(2**8 + n   )
         n += 1
   _lowerCAmelCase					     =		[chr(snake_case_   ) for n in cs]
   return dict(zip(snake_case_   ,	snake_case_   )   )






def 	__UpperCAmelCase					(       snake_case_  :  Dict   )							->	List[Any]:






   """simple docstring"""


   _lowerCAmelCase					     =		set()
   _lowerCAmelCase					     =		word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)   )
      _lowerCAmelCase					     =		char
   return pairs



class       __lowerCamelCase	(   __lowercase    ):
   __UpperCamelCase									=   VOCAB_FILES_NAMES
   __UpperCamelCase									=   PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase									=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase									=   ['input_ids', 'attention_mask']







   def __init__(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase="replace"       ,	lowerCamelCase="<s>"       ,	lowerCamelCase="</s>"       ,	lowerCamelCase="</s>"       ,	lowerCamelCase="<s>"       ,	lowerCamelCase="<unk>"       ,	lowerCamelCase="<pad>"       ,	lowerCamelCase="<mask>"       ,	lowerCamelCase=False       ,	**lowerCamelCase       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else bos_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else eos_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else sep_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else cls_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else unk_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else mask_token

      super().__init__(
          errors=lowerCamelCase       ,	bos_token=lowerCamelCase       ,	eos_token=lowerCamelCase       ,	unk_token=lowerCamelCase       ,	sep_token=lowerCamelCase       ,	cls_token=lowerCamelCase       ,	pad_token=lowerCamelCase       ,	mask_token=lowerCamelCase       ,	add_prefix_space=lowerCamelCase       ,	**lowerCamelCase       ,	)
      with open(lowerCamelCase       ,	encoding="""utf-8"""				) as vocab_handle:
         _lowerCAmelCase					     =		json.load(lowerCamelCase				)
      _lowerCAmelCase					     =		{v: k for k, v in self.encoder.items()}
      _lowerCAmelCase					     =		errors  # how to handle errors in decoding
      _lowerCAmelCase					     =		bytes_to_unicode()
      _lowerCAmelCase					     =		{v: k for k, v in self.byte_encoder.items()}
      with open(lowerCamelCase       ,	encoding="""utf-8"""				) as merges_handle:
         _lowerCAmelCase					     =		merges_handle.read().split("""\n"""				)[1:-1]
      _lowerCAmelCase					     =		[tuple(merge.split()				) for merge in bpe_merges]
      _lowerCAmelCase					     =		dict(zip(lowerCamelCase       ,	range(len(lowerCamelCase				)				)				)				)
      _lowerCAmelCase					     =		{}
      _lowerCAmelCase					     =		add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      _lowerCAmelCase					     =		re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)







   @property
   def 							A__						(self				):




      '''simple docstring'''




      return len(self.encoder				)







   def 							A__						(self				):




      '''simple docstring'''




      return dict(self.encoder       ,	**self.added_tokens_encoder				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      if token in self.cache:
         return self.cache[token]
      _lowerCAmelCase					     =		tuple(lowerCamelCase				)
      _lowerCAmelCase					     =		get_pairs(lowerCamelCase				)

      if not pairs:
         return token

      while True:
         _lowerCAmelCase					     =		min(lowerCamelCase       ,	key=lambda lowerCamelCase				: self.bpe_ranks.get(lowerCamelCase       ,	float("""inf"""				)				)				)
         if bigram not in self.bpe_ranks:
            break
         _lowerCAmelCase     ,						_lowerCAmelCase					     =		bigram
         _lowerCAmelCase					     =		[]
         _lowerCAmelCase					     =		0
         while i < len(lowerCamelCase				):
            try:
               _lowerCAmelCase					     =		word.index(lowerCamelCase       ,	lowerCamelCase				)
            except ValueError:
               new_word.extend(word[i:]				)
               break
            else:
               new_word.extend(word[i:j]				)
               _lowerCAmelCase					     =		j

            if word[i] == first and i < len(lowerCamelCase				) - 1 and word[i + 1] == second:
               new_word.append(first + second				)
               i += 2
            else:
               new_word.append(word[i]				)
               i += 1
         _lowerCAmelCase					     =		tuple(lowerCamelCase				)
         _lowerCAmelCase					     =		new_word
         if len(lowerCamelCase				) == 1:
            break
         else:
            _lowerCAmelCase					     =		get_pairs(lowerCamelCase				)
      _lowerCAmelCase					     =		""" """.join(lowerCamelCase				)
      _lowerCAmelCase					     =		word
      return word







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[]
      for token in re.findall(self.pat       ,	lowerCamelCase				):
         _lowerCAmelCase					     =		"""""".join(
             self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase				).split(""" """				)				)
      return bpe_tokens







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      return self.encoder.get(lowerCamelCase       ,	self.encoder.get(self.unk_token				)				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      return self.decoder.get(lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		"""""".join(lowerCamelCase				)
      _lowerCAmelCase					     =		bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8"""       ,	errors=self.errors				)
      return text







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None				):




      '''simple docstring'''




      if not os.path.isdir(lowerCamelCase				):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
         return
      _lowerCAmelCase					     =		os.path.join(
          lowerCamelCase       ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
      _lowerCAmelCase					     =		os.path.join(
          lowerCamelCase       ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

      with open(lowerCamelCase       ,	"""w"""       ,	encoding="""utf-8"""				) as f:
         f.write(json.dumps(self.encoder       ,	indent=2       ,	sort_keys=lowerCamelCase       ,	ensure_ascii=lowerCamelCase				) + """\n"""				)

      _lowerCAmelCase					     =		0
      with open(lowerCamelCase       ,	"""w"""       ,	encoding="""utf-8"""				) as writer:
         writer.write("""#version: 0.2\n"""				)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,	key=lambda lowerCamelCase				: kv[1]				):
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                   """ Please check that the tokenizer is not corrupted!"""				)
               _lowerCAmelCase					     =		token_index
            writer.write(""" """.join(lowerCamelCase				) + """\n"""				)
            index += 1

      return vocab_file, merge_file







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None				):




      '''simple docstring'''




      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowerCAmelCase					     =		[self.cls_token_id]
      _lowerCAmelCase					     =		[self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = False				):




      '''simple docstring'''




      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase       ,	token_ids_a=lowerCamelCase       ,	already_has_special_tokens=lowerCamelCase				)

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase				)) + [1]
      return [1] + ([0] * len(lowerCamelCase				)) + [1, 1] + ([0] * len(lowerCamelCase				)) + [1]







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[self.sep_token_id]
      _lowerCAmelCase					     =		[self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep				) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase=False       ,	**lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		kwargs.pop("""add_prefix_space"""       ,	self.add_prefix_space				)
      if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase				) > 0 and not text[0].isspace()):
         _lowerCAmelCase					     =		""" """ + text
      return (text, kwargs)