import unittest

from transformers import ElectraConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    ElectraForCausalLM,
					    ElectraForMaskedLM,
					    ElectraForMultipleChoice,
					    ElectraForPreTraining,
					    ElectraForQuestionAnswering,
					    ElectraForSequenceClassification,
					    ElectraForTokenClassification,
					    ElectraModel,
					)
					from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST
class SCREAMING_SNAKE_CASE_		:







			def __init__(    self	:	Union[str, Any]      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	str=13      ,      lowerCamelCase_	:	Union[str, Any]=7      ,      lowerCamelCase_	:	Union[str, Any]=True      ,      lowerCamelCase_	:	int=True      ,      lowerCamelCase_	:	Dict=True      ,      lowerCamelCase_	:	Optional[int]=True      ,      lowerCamelCase_	:	List[str]=99      ,      lowerCamelCase_	:	Union[str, Any]=32      ,      lowerCamelCase_	:	str=5      ,      lowerCamelCase_	:	List[Any]=4      ,      lowerCamelCase_	:	Optional[int]=37      ,      lowerCamelCase_	:	Tuple="gelu"      ,      lowerCamelCase_	:	Dict=0.1      ,      lowerCamelCase_	:	str=0.1      ,      lowerCamelCase_	:	Dict=512      ,      lowerCamelCase_	:	Dict=16      ,      lowerCamelCase_	:	List[Any]=2      ,      lowerCamelCase_	:	Dict=0.0_2      ,      lowerCamelCase_	:	int=3      ,      lowerCamelCase_	:	Union[str, Any]=4      ,      lowerCamelCase_	:	Union[str, Any]=None      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			parent
				UpperCamelCase      =			batch_size
				UpperCamelCase      =			seq_length
				UpperCamelCase      =			is_training
				UpperCamelCase      =			use_input_mask
				UpperCamelCase      =			use_token_type_ids
				UpperCamelCase      =			use_labels
				UpperCamelCase      =			vocab_size
				UpperCamelCase      =			hidden_size
				UpperCamelCase      =			num_hidden_layers
				UpperCamelCase      =			num_attention_heads
				UpperCamelCase      =			intermediate_size
				UpperCamelCase      =			hidden_act
				UpperCamelCase      =			hidden_dropout_prob
				UpperCamelCase      =			attention_probs_dropout_prob
				UpperCamelCase      =			max_position_embeddings
				UpperCamelCase      =			type_vocab_size
				UpperCamelCase      =			type_sequence_label_size
				UpperCamelCase      =			initializer_range
				UpperCamelCase      =			num_labels
				UpperCamelCase      =			num_choices
				UpperCamelCase      =			scope







			def 	lowerCamelCase_  (    self	:	Optional[int]							):





				"""simple docstring"""


				UpperCamelCase      =			ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size							)

				UpperCamelCase      =			None
				if self.use_input_mask:
					UpperCamelCase      =			random_attention_mask([self.batch_size, self.seq_length]							)

				UpperCamelCase      =			None
				if self.use_token_type_ids:
					UpperCamelCase      =			ids_tensor([self.batch_size, self.seq_length]      ,      self.type_vocab_size							)

				UpperCamelCase      =			None
				UpperCamelCase      =			None
				UpperCamelCase      =			None
				if self.use_labels:
					UpperCamelCase      =			ids_tensor([self.batch_size]      ,      self.type_sequence_label_size							)
					UpperCamelCase      =			ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels							)
					UpperCamelCase      =			ids_tensor([self.batch_size]      ,      self.num_choices							)
					UpperCamelCase      =			ids_tensor([self.batch_size, self.seq_length]      ,      1							)

				UpperCamelCase      =			self.get_config()

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    fake_token_labels,
				)







			def 	lowerCamelCase_  (    self	:	Tuple							):





				"""simple docstring"""


				return ElectraConfig(
				    vocab_size=self.vocab_size      ,      hidden_size=self.hidden_size      ,      num_hidden_layers=self.num_hidden_layers      ,      num_attention_heads=self.num_attention_heads      ,      intermediate_size=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      max_position_embeddings=self.max_position_embeddings      ,      type_vocab_size=self.type_vocab_size      ,      is_decoder=lowerCamelCase_      ,      initializer_range=self.initializer_range      ,      )







			def 	lowerCamelCase_  (    self	:	List[Any]							):





				"""simple docstring"""


				(
				    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,        
				)      =			self.prepare_config_and_inputs()

				UpperCamelCase      =			True
				UpperCamelCase      =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]							)
				UpperCamelCase      =			ids_tensor([self.batch_size, self.seq_length]      ,      vocab_size=2							)

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    encoder_hidden_states,
				    encoder_attention_mask,
				)







			def 	lowerCamelCase_  (    self	:	List[str]      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	str      ,      lowerCamelCase_	:	str      ,      lowerCamelCase_	:	List[str]      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Tuple      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			ElectraModel(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_							)
				UpperCamelCase      =			model(lowerCamelCase_      ,      token_type_ids=lowerCamelCase_							)
				UpperCamelCase      =			model(lowerCamelCase_							)
				self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)							)







			def 	lowerCamelCase_  (    self	:	List[str]      ,      lowerCamelCase_	:	Optional[int]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	Tuple      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Optional[int]      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			True
				UpperCamelCase      =			ElectraModel(lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(
				    lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      encoder_hidden_states=lowerCamelCase_      ,      encoder_attention_mask=lowerCamelCase_      ,      )
				UpperCamelCase      =			model(
				    lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      encoder_hidden_states=lowerCamelCase_      ,      )
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_							)
				self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)							)







			def 	lowerCamelCase_  (    self	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Optional[Any]      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			ElectraForMaskedLM(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_							)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)							)







			def 	lowerCamelCase_  (    self	:	Any      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	int      ,      lowerCamelCase_	:	List[str]      ,      lowerCamelCase_	:	Tuple      ,      lowerCamelCase_	:	Tuple      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			ElectraForCausalLM(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_							)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)							)







			def 	lowerCamelCase_  (    self	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Tuple      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	List[str]      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	int      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			self.num_labels
				UpperCamelCase      =			ElectraForTokenClassification(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_							)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.num_labels)							)







			def 	lowerCamelCase_  (    self	:	Optional[Any]      ,      lowerCamelCase_	:	List[str]      ,      lowerCamelCase_	:	List[str]      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	Optional[int]      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	int      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Any      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			self.num_labels
				UpperCamelCase      =			ElectraForPreTraining(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_							)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length)							)







			def 	lowerCamelCase_  (    self	:	int      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Optional[int]      ,      lowerCamelCase_	:	int      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	Dict      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			self.num_labels
				UpperCamelCase      =			ElectraForSequenceClassification(lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_							)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_labels)							)







			def 	lowerCamelCase_  (    self	:	Optional[int]      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	Optional[int]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	List[Any]      ,      lowerCamelCase_	:	int      ,      lowerCamelCase_	:	List[Any]      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			ElectraForQuestionAnswering(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			model(
				    lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      start_positions=lowerCamelCase_      ,      end_positions=lowerCamelCase_      ,      )
				self.parent.assertEqual(result.start_logits.shape      ,      (self.batch_size, self.seq_length)							)
				self.parent.assertEqual(result.end_logits.shape      ,      (self.batch_size, self.seq_length)							)







			def 	lowerCamelCase_  (    self	:	Optional[Any]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Optional[Any]      ,      lowerCamelCase_	:	int      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Union[str, Any]      ,      lowerCamelCase_	:	Dict      ,      ):





				"""simple docstring"""


				UpperCamelCase      =			self.num_choices
				UpperCamelCase      =			ElectraForMultipleChoice(config=lowerCamelCase_							)
				model.to(lowerCamelCase_							)
				model.eval()
				UpperCamelCase      =			input_ids.unsqueeze(1							).expand(-1      ,      self.num_choices      ,      -1							).contiguous()
				UpperCamelCase      =			token_type_ids.unsqueeze(1							).expand(-1      ,      self.num_choices      ,      -1							).contiguous()
				UpperCamelCase      =			input_mask.unsqueeze(1							).expand(-1      ,      self.num_choices      ,      -1							).contiguous()
				UpperCamelCase      =			model(
				    lowerCamelCase_      ,      attention_mask=lowerCamelCase_      ,      token_type_ids=lowerCamelCase_      ,      labels=lowerCamelCase_      ,      )
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_choices)							)







			def 	lowerCamelCase_  (    self	:	List[Any]							):





				"""simple docstring"""


				UpperCamelCase      =			self.prepare_config_and_inputs()
				(
				    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,    (
				    UpperCamelCase    
				)     ,        
				)      =			config_and_inputs
				UpperCamelCase      =			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
				return config, inputs_dict





@require_torch
class SCREAMING_SNAKE_CASE_		(			__lowerCAmelCase     ,    __lowerCAmelCase     ,    unittest.TestCase		):
			__lowerCAmelCase											=							(
			    (
			        ElectraModel,
			        ElectraForPreTraining,
			        ElectraForMaskedLM,
			        ElectraForCausalLM,
			        ElectraForMultipleChoice,
			        ElectraForTokenClassification,
			        ElectraForSequenceClassification,
			        ElectraForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			__lowerCAmelCase											=							(
			    {
			        """feature-extraction""": ElectraModel,
			        """fill-mask""": ElectraForMaskedLM,
			        """question-answering""": ElectraForQuestionAnswering,
			        """text-classification""": ElectraForSequenceClassification,
			        """text-generation""": ElectraForCausalLM,
			        """token-classification""": ElectraForTokenClassification,
			        """zero-shot""": ElectraForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__lowerCAmelCase											=							True







			def 	lowerCamelCase_  (    self	:	Optional[int]      ,      lowerCamelCase_	:	Dict      ,      lowerCamelCase_	:	Any      ,      lowerCamelCase_	:	List[str]=False							):





				"""simple docstring"""


				UpperCamelCase      =			super()._prepare_for_class(lowerCamelCase_      ,      lowerCamelCase_      ,      return_labels=lowerCamelCase_							)

				if return_labels:
					if model_class in get_values(lowerCamelCase_							):
						UpperCamelCase      =			torch.zeros(
						    (self.model_tester.batch_size, self.model_tester.seq_length)      ,      dtype=torch.long      ,      device=lowerCamelCase_							)
				return inputs_dict







			def 	lowerCamelCase_  (    self	:	List[str]							):





				"""simple docstring"""


				UpperCamelCase      =			ElectraModelTester(self							)
				UpperCamelCase      =			ConfigTester(self      ,      config_class=lowerCamelCase_      ,      hidden_size=37							)







			def 	lowerCamelCase_  (    self	:	Any							):





				"""simple docstring"""


				self.config_tester.run_common_tests()







			def 	lowerCamelCase_  (    self	:	List[str]							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_model(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Any							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_electra_model_as_decoder(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Tuple							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
					UpperCamelCase      =			type
					self.model_tester.create_and_check_electra_model(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Tuple							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_masked_lm(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Tuple							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_token_classification(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Dict							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_pretraining(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Any							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_sequence_classification(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	List[str]							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_question_answering(*lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Optional[int]							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_electra_for_multiple_choice(*lowerCamelCase_							)







			@slow
			def 	lowerCamelCase_  (    self	:	Optional[int]							):





				"""simple docstring"""


				for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					UpperCamelCase      =			ElectraModel.from_pretrained(lowerCamelCase_							)
					self.assertIsNotNone(lowerCamelCase_							)







			def 	lowerCamelCase_  (    self	:	Optional[Any]							):





				"""simple docstring"""


				UpperCamelCase      =			self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_electra_for_causal_lm(*lowerCamelCase_							)





@require_torch
class SCREAMING_SNAKE_CASE_		(			unittest.TestCase		):







			@slow
			def 	lowerCamelCase_  (    self	:	Optional[int]							):





				"""simple docstring"""


				UpperCamelCase      =			ElectraModel.from_pretrained("""google/electra-small-discriminator"""							)
				UpperCamelCase      =			torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]							)
				UpperCamelCase      =			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
				UpperCamelCase      =			model(lowerCamelCase_      ,      attention_mask=lowerCamelCase_							)[0]
				UpperCamelCase      =			torch.Size((1, 11, 256)							)
				self.assertEqual(output.shape      ,      lowerCamelCase_							)
				UpperCamelCase      =			torch.tensor(
				    [[[0.4_4_7_1, 0.6_8_2_1, -0.3_2_6_5], [0.4_6_2_7, 0.5_2_5_5, -0.3_6_6_8], [0.4_5_3_2, 0.3_3_1_3, -0.4_3_4_4]]]							)

				self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,      lowerCamelCase_      ,      atol=1E-4							)							)

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   		=			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   		=			{
    """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/config.json""",
    """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/config.json""",
    """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/config.json""",
    """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/config.json""",
    # See all Splinter models at https://huggingface.co/models?filter=splinter
}
class SCREAMING_SNAKE_CASE_		(			__lowerCAmelCase		):
   __lowerCAmelCase											=							"""splinter"""







   def __init__(    self	:	List[Any]      ,      lowerCamelCase_	:	int=3_0522      ,      lowerCamelCase_	:	Any=768      ,      lowerCamelCase_	:	List[Any]=12      ,      lowerCamelCase_	:	Tuple=12      ,      lowerCamelCase_	:	Tuple=3072      ,      lowerCamelCase_	:	Any="gelu"      ,      lowerCamelCase_	:	Union[str, Any]=0.1      ,      lowerCamelCase_	:	str=0.1      ,      lowerCamelCase_	:	int=512      ,      lowerCamelCase_	:	List[Any]=2      ,      lowerCamelCase_	:	List[Any]=0.0_2      ,      lowerCamelCase_	:	Dict=1E-12      ,      lowerCamelCase_	:	Tuple=True      ,      lowerCamelCase_	:	str=0      ,      lowerCamelCase_	:	Any=104      ,      **lowerCamelCase_	:	Dict      ,      ):





    """simple docstring"""


    super().__init__(pad_token_id=lowerCamelCase_      ,      **lowerCamelCase_							)

    UpperCamelCase      =			vocab_size
    UpperCamelCase      =			max_position_embeddings
    UpperCamelCase      =			hidden_size
    UpperCamelCase      =			num_hidden_layers
    UpperCamelCase      =			num_attention_heads
    UpperCamelCase      =			intermediate_size
    UpperCamelCase      =			hidden_act
    UpperCamelCase      =			hidden_dropout_prob
    UpperCamelCase      =			attention_probs_dropout_prob
    UpperCamelCase      =			initializer_range
    UpperCamelCase      =			type_vocab_size
    UpperCamelCase      =			layer_norm_eps
    UpperCamelCase      =			use_cache
    UpperCamelCase      =			question_token_id
