


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    RemBertConfig,
			    RemBertForCausalLM,
			    RemBertForMaskedLM,
			    RemBertForMultipleChoice,
			    RemBertForQuestionAnswering,
			    RemBertForSequenceClassification,
			    RemBertForTokenClassification,
			    RemBertModel,
			)
			from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class     snake_case__  :
				def __init__(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase=13     ,      _UpperCAmelCase=7     ,      _UpperCAmelCase=True     ,      _UpperCAmelCase=True     ,      _UpperCAmelCase=True     ,      _UpperCAmelCase=True     ,      _UpperCAmelCase=99     ,      _UpperCAmelCase=32     ,      _UpperCAmelCase=18     ,      _UpperCAmelCase=43     ,      _UpperCAmelCase=5     ,      _UpperCAmelCase=4     ,      _UpperCAmelCase=37     ,      _UpperCAmelCase="gelu"     ,      _UpperCAmelCase=0.1     ,      _UpperCAmelCase=0.1     ,      _UpperCAmelCase=512     ,      _UpperCAmelCase=16     ,      _UpperCAmelCase=2     ,      _UpperCAmelCase=0.02     ,      _UpperCAmelCase=3     ,      _UpperCAmelCase=4     ,      _UpperCAmelCase=None     ,      ):




										"""simple docstring"""




										__magic_name__				:							Dict				=      parent
										__magic_name__				:							Optional[Any]				=      batch_size
										__magic_name__				:							Optional[int]				=      seq_length
										__magic_name__				:							int				=      is_training
										__magic_name__				:							int				=      use_input_mask
										__magic_name__				:							Optional[Any]				=      use_token_type_ids
										__magic_name__				:							int				=      use_labels
										__magic_name__				:							Any				=      vocab_size
										__magic_name__				:							Optional[Any]				=      hidden_size
										__magic_name__				:							List[str]				=      input_embedding_size
										__magic_name__				:							Optional[Any]				=      output_embedding_size
										__magic_name__				:							Tuple				=      num_hidden_layers
										__magic_name__				:							Tuple				=      num_attention_heads
										__magic_name__				:							int				=      intermediate_size
										__magic_name__				:							Union[str, Any]				=      hidden_act
										__magic_name__				:							Tuple				=      hidden_dropout_prob
										__magic_name__				:							Any				=      attention_probs_dropout_prob
										__magic_name__				:							Dict				=      max_position_embeddings
										__magic_name__				:							Dict				=      type_vocab_size
										__magic_name__				:							Tuple				=      type_sequence_label_size
										__magic_name__				:							Optional[int]				=      initializer_range
										__magic_name__				:							List[str]				=      num_labels
										__magic_name__				:							List[Any]				=      num_choices
										__magic_name__				:							Union[str, Any]				=      scope
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Any				=      ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size      )

										__magic_name__				:							Optional[Any]				=      None
										if self.use_input_mask:
																__magic_name__				:							int				=      random_attention_mask([self.batch_size, self.seq_length]      )

										__magic_name__				:							Tuple				=      None
										if self.use_token_type_ids:
																__magic_name__				:							Optional[int]				=      ids_tensor([self.batch_size, self.seq_length]     ,      self.type_vocab_size      )

										__magic_name__				:							List[str]				=      None
										__magic_name__				:							Any				=      None
										__magic_name__				:							Union[str, Any]				=      None
										if self.use_labels:
																__magic_name__				:							int				=      ids_tensor([self.batch_size]     ,      self.type_sequence_label_size      )
																__magic_name__				:							Tuple				=      ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels      )
																__magic_name__				:							Optional[Any]				=      ids_tensor([self.batch_size]     ,      self.num_choices      )

										__magic_name__				:							Optional[int]				=      RemBertConfig(
										    vocab_size=self.vocab_size     ,      hidden_size=self.hidden_size     ,      input_embedding_size=self.input_embedding_size     ,      output_embedding_size=self.output_embedding_size     ,      num_hidden_layers=self.num_hidden_layers     ,      num_attention_heads=self.num_attention_heads     ,      intermediate_size=self.intermediate_size     ,      hidden_act=self.hidden_act     ,      hidden_dropout_prob=self.hidden_dropout_prob     ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,      max_position_embeddings=self.max_position_embeddings     ,      type_vocab_size=self.type_vocab_size     ,      is_decoder=_UpperCAmelCase     ,      initializer_range=self.initializer_range     ,      )

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										(
										    (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, 
										)				:							Tuple				=      self.prepare_config_and_inputs()

										__magic_name__				:							Optional[int]				=      True
										__magic_name__				:							Dict				=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
										__magic_name__				:							Any				=      ids_tensor([self.batch_size, self.seq_length]     ,      vocab_size=2      )

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      RemBertModel(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							Optional[int]				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase      )
										__magic_name__				:							str				=      model(_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase      )
										__magic_name__				:							Dict				=      model(_UpperCAmelCase      )
										self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      ):




										"""simple docstring"""




										__magic_name__				:							Any				=      True
										__magic_name__				:							Any				=      RemBertModel(_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							Dict				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      encoder_hidden_states=_UpperCAmelCase     ,      encoder_attention_mask=_UpperCAmelCase     ,      )
										__magic_name__				:							Optional[Any]				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      encoder_hidden_states=_UpperCAmelCase     ,      )
										__magic_name__				:							Union[str, Any]				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase      )
										self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      ):




										"""simple docstring"""




										__magic_name__				:							Union[str, Any]				=      RemBertForCausalLM(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							Tuple				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      labels=_UpperCAmelCase      )
										self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Dict				=      RemBertForMaskedLM(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							int				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      labels=_UpperCAmelCase      )
										self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      ):




										"""simple docstring"""




										__magic_name__				:							int				=      True
										__magic_name__				:							Optional[Any]				=      True
										__magic_name__				:							str				=      RemBertForCausalLM(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()

										# first forward pass
										__magic_name__				:							Union[str, Any]				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      encoder_hidden_states=_UpperCAmelCase     ,      encoder_attention_mask=_UpperCAmelCase     ,      use_cache=_UpperCAmelCase     ,      )
										__magic_name__				:							Optional[Any]				=      outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										__magic_name__				:							Optional[Any]				=      ids_tensor((self.batch_size, 3)     ,      config.vocab_size      )
										__magic_name__				:							str				=      ids_tensor((self.batch_size, 3)     ,      vocab_size=2      )

										# append to next input_ids and
										__magic_name__				:							Tuple				=      torch.cat([input_ids, next_tokens]     ,      dim=-1      )
										__magic_name__				:							Tuple				=      torch.cat([input_mask, next_mask]     ,      dim=-1      )

										__magic_name__				:							List[str]				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      encoder_hidden_states=_UpperCAmelCase     ,      encoder_attention_mask=_UpperCAmelCase     ,      output_hidden_states=_UpperCAmelCase     ,      )['hidden_states'][0]
										__magic_name__				:							Dict				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      encoder_hidden_states=_UpperCAmelCase     ,      encoder_attention_mask=_UpperCAmelCase     ,      past_key_values=_UpperCAmelCase     ,      output_hidden_states=_UpperCAmelCase     ,      )['hidden_states'][0]

										# select random slice
										__magic_name__				:							int				=      ids_tensor((1,)     ,      output_from_past.shape[-1]      ).item()
										__magic_name__				:							int				=      output_from_no_past[:, -3:, random_slice_idx].detach()
										__magic_name__				:							str				=      output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(_UpperCAmelCase     ,      _UpperCAmelCase     ,      atol=1E-3      )      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Optional[int]				=      RemBertForQuestionAnswering(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							int				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      start_positions=_UpperCAmelCase     ,      end_positions=_UpperCAmelCase     ,      )
										self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)      )
										self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Optional[int]				=      self.num_labels
										__magic_name__				:							Optional[Any]				=      RemBertForSequenceClassification(_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							Optional[int]				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      labels=_UpperCAmelCase      )
										self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_labels)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Any				=      self.num_labels
										__magic_name__				:							Union[str, Any]				=      RemBertForTokenClassification(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							str				=      model(_UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      labels=_UpperCAmelCase      )
										self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							List[str]				=      self.num_choices
										__magic_name__				:							Optional[Any]				=      RemBertForMultipleChoice(config=_UpperCAmelCase      )
										model.to(_UpperCAmelCase      )
										model.eval()
										__magic_name__				:							str				=      input_ids.unsqueeze(1      ).expand(-1     ,      self.num_choices     ,      -1      ).contiguous()
										__magic_name__				:							str				=      token_type_ids.unsqueeze(1      ).expand(-1     ,      self.num_choices     ,      -1      ).contiguous()
										__magic_name__				:							Optional[int]				=      input_mask.unsqueeze(1      ).expand(-1     ,      self.num_choices     ,      -1      ).contiguous()
										__magic_name__				:							List[str]				=      model(
										    _UpperCAmelCase     ,      attention_mask=_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      labels=_UpperCAmelCase     ,      )
										self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)      )





				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							str				=      self.prepare_config_and_inputs()
										(
										    (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, 
										)				:							Optional[Any]				=      config_and_inputs
										__magic_name__				:							Union[str, Any]				=      {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
										return config, inputs_dict






@require_torch
class     snake_case__  (      _A						,       _A						,       unittest.TestCase  ):
				__lowerCAmelCase		=      (
				    (
				        RemBertModel,
				        RemBertForMaskedLM,
				        RemBertForCausalLM,
				        RemBertForMultipleChoice,
				        RemBertForQuestionAnswering,
				        RemBertForSequenceClassification,
				        RemBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				__lowerCAmelCase		=      (RemBertForCausalLM,) if is_torch_available() else ()
				__lowerCAmelCase		=      (
				    {
				        '''feature-extraction''': RemBertModel,
				        '''fill-mask''': RemBertForMaskedLM,
				        '''question-answering''': RemBertForQuestionAnswering,
				        '''text-classification''': RemBertForSequenceClassification,
				        '''text-generation''': RemBertForCausalLM,
				        '''token-classification''': RemBertForTokenClassification,
				        '''zero-shot''': RemBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Dict				=      RemBertModelTester(self      )
										__magic_name__				:							Tuple				=      ConfigTester(self     ,      config_class=_UpperCAmelCase     ,      hidden_size=37      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.config_tester.run_common_tests()
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																__magic_name__				:							int				=      type
																self.model_tester.create_and_check_model(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							List[Any]				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Any				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							int				=      self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_decoder_model_past_large_inputs(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							List[str]				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Dict				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Union[str, Any]				=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							Any				=      self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_model_as_decoder(*_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										(
										    (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, (
										    __magic_name__
										)		, 
										)				:							str				=      self.model_tester.prepare_config_and_inputs_for_decoder()

										__magic_name__				:							Optional[int]				=      None

										self.model_tester.create_and_check_model_as_decoder(
										    _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      )





				@slow
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__magic_name__				:							Dict				=      RemBertModel.from_pretrained(_UpperCAmelCase      )
																self.assertIsNotNone(_UpperCAmelCase      )






@require_torch
class     snake_case__  (      unittest.TestCase  ):
				@slow
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							str				=      RemBertModel.from_pretrained('google/rembert'      )
										__magic_name__				:							Tuple				=      torch.tensor([[312, 5_6498, 313, 2125, 313]]      )
										__magic_name__				:							List[str]				=      torch.tensor([[0, 0, 0, 1, 1]]      )
										with torch.no_grad():
																__magic_name__				:							Dict				=      model(_UpperCAmelCase     ,      token_type_ids=_UpperCAmelCase     ,      output_hidden_states=_UpperCAmelCase      )

										__magic_name__				:							str				=      1152

										__magic_name__				:							List[Any]				=      torch.Size((1, 5, hidden_size)      )
										self.assertEqual(output['last_hidden_state'].shape     ,      _UpperCAmelCase      )

										__magic_name__				:							List[str]				=      torch.tensor(
										    [
										        [
										            [0.0754, -0.2022, 0.1904],
										            [-0.3354, -0.3692, -0.4791],
										            [-0.2314, -0.6729, -0.0749],
										            [-0.0396, -0.3105, -0.4234],
										            [-0.1571, -0.0525, 0.5353],
										        ]
										    ]      )

										# Running on the original tf implementation gives slightly different results here.
										# Not clear why this variations is present
										# TODO: Find reason for discrepancy
										# expected_original_implementation = [[
										#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
										#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
										#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
										#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
										#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
										# ]]

										self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]     ,      _UpperCAmelCase     ,      atol=1E-4      )      )






import dataclasses
import enum
import functools
import math
import re

# The following script is adapted from the script of TaPas.
# Original: https://github.com/google-research/tapas/master/wikisql_utils.py
from typing import Any, List, Text


snake_case__     							= 'none'
snake_case__     							= 'none'
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :List[str]      ,						lowercase__  :Optional[Any]				)       ->  List[str]:
						__magic_name__				:							Dict				=      value.split(lowercase__				)
						return len(lowercase__				) > 1 and any((len(lowercase__				) == 3 for x in split)				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :List[Any]				)       ->  List[str]:
						if isinstance(lowercase__      ,						lowercase__				):
												return value
						if isinstance(lowercase__      ,						lowercase__				):
												return float(lowercase__				)
						if not isinstance(lowercase__      ,						lowercase__				):
												raise ValueError('Argument value is not a string. Can\'t parse it as float'				)
						__magic_name__				:							List[str]				=      value

						try:
												# Example: 1,000.7
												if "." in sanitized and "," in sanitized:
																		return float(sanitized.replace(','      ,						''				)				)
												# 1,000
												if "," in sanitized and _split_thousands(','      ,						lowercase__				):
																		return float(sanitized.replace(','      ,						''				)				)
												# 5,5556
												if "," in sanitized and sanitized.count(','				) == 1 and not _split_thousands(','      ,						lowercase__				):
																		return float(sanitized.replace(','      ,						'.'				)				)
												# 0.0.0.1
												if sanitized.count('.'				) > 1:
																		return float(sanitized.replace('.'      ,						''				)				)
												# 0,0,0,1
												if sanitized.count(','				) > 1:
																		return float(sanitized.replace(','      ,						''				)				)
												return float(lowercase__				)
						except ValueError:
												# Avoid adding the sanitized value in the error message.
												raise ValueError('Unable to convert value to float'				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple				)       ->  Tuple:
						if answer is None:
												return None
						try:
												__magic_name__				:							Any				=      convert_to_float(lowercase__				)
												if isinstance(lowercase__      ,						lowercase__				) and math.isnan(lowercase__				):
																		return None
												return value
						except ValueError:
												return answer.lower()


snake_case__     							= {
    'text': lambda x: x,
    'real': convert_to_float,
}



class     snake_case__  (      enum.Enum  ):
				__lowerCAmelCase		=      0
				__lowerCAmelCase		=      1
				__lowerCAmelCase		=      2
				__lowerCAmelCase		=      3
				__lowerCAmelCase		=      4
				__lowerCAmelCase		=      5






class     snake_case__  (      enum.Enum  ):
				__lowerCAmelCase		=      0
				__lowerCAmelCase		=      1
				__lowerCAmelCase		=      2






@dataclasses.dataclass
class     snake_case__  :
				__lowerCAmelCase		=      42
				__lowerCAmelCase		=      42
				__lowerCAmelCase		=      42






snake_case__     							= re.compile(R'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Any				)       ->  str:
						return list(_TOKENIZER.findall(x.lower()				)				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :str      ,						lowercase__  :str      ,						lowercase__  :Dict				)       ->  Dict:
						if operator == _Operator.EQUALS:
												return src == tgt
						elif operator == _Operator.GREATER:
												return src > tgt
						elif operator == _Operator.LESSER:
												return src < tgt
						raise ValueError(f"Unknown operator: {operator}"				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :List[Any]      ,						lowercase__  :Tuple      ,						lowercase__  :Union[str, Any]				)       ->  Optional[int]:
						__magic_name__				:							str				=      table['types']
						return _TYPE_CONVERTER[types[column]](lowercase__				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple				)       ->  Union[str, Any]:
						return isinstance(lowercase__      ,						lowercase__				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :List[Any]      ,						lowercase__  :List[str]      ,						lowercase__  :int				)       ->  str:
						for cond in conditions:
												__magic_name__				:							Optional[Any]				=      row[cond.column]

												__magic_name__				:							List[str]				=      _parse_value(lowercase__      ,						cond.column      ,						cond.cmp_value				)

												if _is_string(lowercase__				) and _is_string(lowercase__				):
																		__magic_name__				:							str				=      _normalize_for_match(lowercase__				)
																		__magic_name__				:							List[str]				=      _normalize_for_match(lowercase__				)

												if not isinstance(lowercase__      ,						type(lowercase__				)				):
																		raise ValueError('Type difference {} != {}'.format(type(lowercase__				)      ,						type(lowercase__				)				)				)

												if not _compare(cond.operator      ,						lowercase__      ,						lowercase__				):
																		return False
						return True
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :str      ,						lowercase__  :List[Any]      ,						lowercase__  :List[Any]				)       ->  int:
						if not answer_coordinates:
												if aggregation_op == _Aggregation.COUNT:
																		return 0.0
												else:
																		return EMPTY_ANSWER_AGG

    # Count can support non numeric answers.
						if aggregation_op == _Aggregation.COUNT:
												return float(len(lowercase__				)				)

						# If we have just one answer, if float returns it or try a conversion.
						__magic_name__				:							Any				=      [table['rows'][i][j] for (i, j) in answer_coordinates]
						if len(lowercase__				) == 1:
												try:
																		return convert_to_float(values[0]				)
												except ValueError as e:
																		if aggregation_op != _Aggregation.NONE:
																								raise e

						if aggregation_op == _Aggregation.NONE:
												return None

						# Other aggregation only support numeric values. Bail out if we have strings.
						if not all((isinstance(lowercase__      ,						(int, float)				) for v in values)				):
												return None

						if aggregation_op == _Aggregation.SUM:
												return float(sum(lowercase__				)				)
						elif aggregation_op == _Aggregation.AVERAGE:
												return sum(lowercase__				) / len(lowercase__				)
						else:
												raise ValueError(f"Unknown aggregation: {aggregation_op}"				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple      ,						lowercase__  :Any				)       ->  Optional[int]:
						__magic_name__				:							Optional[Any]				=      sql_query['agg']
						if aggregation_op_index >= 3:
												__magic_name__				:							Any				=      _Aggregation(lowercase__				)
						else:
												__magic_name__				:							int				=      _Aggregation.NONE

						__magic_name__				:							str				=      sql_query['sel']
						__magic_name__				:							Any				=      [
						    _Condition(lowercase__      ,						_Operator(lowercase__				)      ,						lowercase__				)
						    for column, operator, cmp_value in zip(
						        sql_query['conds']['column_index']      ,						sql_query['conds']['operator_index']      ,						sql_query['conds']['condition']				)
						]

						__magic_name__				:							Any				=      []
						for row in range(len(table['rows']				)				):
												if _respect_conditions(lowercase__      ,						table['rows'][row]      ,						lowercase__				):
																		indices.append((row, target_column)				)

						if not indices:
												return [], aggregation_op

						if len(lowercase__				) == 1:
												return indices, aggregation_op

						# Parsing of MIN/MAX.
						if aggregation_op_index in (1, 2):
												__magic_name__				:							str				=      {2: min, 1: max}
												__magic_name__				:							Optional[int]				=      [(table['rows'][i][j], index) for index, (i, j) in enumerate(lowercase__				)]
												__magic_name__				:							str				=      functools.reduce(operators[sql_query['agg']]      ,						lowercase__				)

												__magic_name__				:							List[Any]				=      [indices[reduced[1]]]
												return ret, _Aggregation.NONE

						return indices, aggregation_op
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple      ,						lowercase__  :Any      ,						lowercase__  :Optional[int]				)       ->  List[Any]:
						if float_answer is not None:
												return [str(lowercase__				)]
						return [str(table['real_rows'][r][c]				) for r, c in answer_coordinates]

def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Optional[Any]      ,						lowercase__  :Union[str, Any]				)       ->  List:
						__magic_name__		, __magic_name__				:							List[Any]				=      _get_answer_coordinates(lowercase__      ,						lowercase__				)
						__magic_name__				:							List[Any]				=      _get_float_answer(lowercase__      ,						lowercase__      ,						lowercase__				)
						__magic_name__				:							Any				=      _get_answer_text(lowercase__      ,						lowercase__      ,						lowercase__				)
						# keep the original data the same with TaPas
						if len(lowercase__				) == 0:
												__magic_name__				:							int				=      [EMPTY_ANSWER]
						return answer_text


