
"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowercase		:						int =					{
    'auto_factory': ['get_values'],
    'configuration_auto': ['ALL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CONFIG_MAPPING', 'MODEL_NAMES_MAPPING', 'AutoConfig'],
    'feature_extraction_auto': ['FEATURE_EXTRACTOR_MAPPING', 'AutoFeatureExtractor'],
    'image_processing_auto': ['IMAGE_PROCESSOR_MAPPING', 'AutoImageProcessor'],
    'processing_auto': ['PROCESSOR_MAPPING', 'AutoProcessor'],
    'tokenization_auto': ['TOKENIZER_MAPPING', 'AutoTokenizer'],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase		:						Any =					[
			    'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_AUDIO_XVECTOR_MAPPING',
			    'MODEL_FOR_BACKBONE_MAPPING',
			    'MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING',
			    'MODEL_FOR_CAUSAL_LM_MAPPING',
			    'MODEL_FOR_CTC_MAPPING',
			    'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING',
			    'MODEL_FOR_DEPTH_ESTIMATION_MAPPING',
			    'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING',
			    'MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING',
			    'MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING',
			    'MODEL_FOR_MASKED_LM_MAPPING',
			    'MODEL_FOR_MASK_GENERATION_MAPPING',
			    'MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
			    'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
			    'MODEL_FOR_OBJECT_DETECTION_MAPPING',
			    'MODEL_FOR_PRETRAINING_MAPPING',
			    'MODEL_FOR_QUESTION_ANSWERING_MAPPING',
			    'MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING',
			    'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
			    'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
			    'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING',
			    'MODEL_FOR_TEXT_ENCODING_MAPPING',
			    'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING',
			    'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_VISION_2_SEQ_MAPPING',
			    'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING',
			    'MODEL_MAPPING',
			    'MODEL_WITH_LM_HEAD_MAPPING',
			    'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING',
			    'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING',
			    'AutoModel',
			    'AutoBackbone',
			    'AutoModelForAudioClassification',
			    'AutoModelForAudioFrameClassification',
			    'AutoModelForAudioXVector',
			    'AutoModelForCausalLM',
			    'AutoModelForCTC',
			    'AutoModelForDepthEstimation',
			    'AutoModelForImageClassification',
			    'AutoModelForImageSegmentation',
			    'AutoModelForInstanceSegmentation',
			    'AutoModelForMaskGeneration',
			    'AutoModelForTextEncoding',
			    'AutoModelForMaskedImageModeling',
			    'AutoModelForMaskedLM',
			    'AutoModelForMultipleChoice',
			    'AutoModelForNextSentencePrediction',
			    'AutoModelForObjectDetection',
			    'AutoModelForPreTraining',
			    'AutoModelForQuestionAnswering',
			    'AutoModelForSemanticSegmentation',
			    'AutoModelForSeq2SeqLM',
			    'AutoModelForSequenceClassification',
			    'AutoModelForSpeechSeq2Seq',
			    'AutoModelForTableQuestionAnswering',
			    'AutoModelForTokenClassification',
			    'AutoModelForUniversalSegmentation',
			    'AutoModelForVideoClassification',
			    'AutoModelForVision2Seq',
			    'AutoModelForVisualQuestionAnswering',
			    'AutoModelForDocumentQuestionAnswering',
			    'AutoModelWithLMHead',
			    'AutoModelForZeroShotImageClassification',
			    'AutoModelForZeroShotObjectDetection',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase		:						str =					[
			    'TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
			    'TF_MODEL_FOR_CAUSAL_LM_MAPPING',
			    'TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
			    'TF_MODEL_FOR_MASK_GENERATION_MAPPING',
			    'TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING',
			    'TF_MODEL_FOR_MASKED_LM_MAPPING',
			    'TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
			    'TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
			    'TF_MODEL_FOR_PRETRAINING_MAPPING',
			    'TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING',
			    'TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING',
			    'TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING',
			    'TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
			    'TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
			    'TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
			    'TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING',
			    'TF_MODEL_FOR_TEXT_ENCODING_MAPPING',
			    'TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
			    'TF_MODEL_FOR_VISION_2_SEQ_MAPPING',
			    'TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING',
			    'TF_MODEL_MAPPING',
			    'TF_MODEL_WITH_LM_HEAD_MAPPING',
			    'TFAutoModel',
			    'TFAutoModelForAudioClassification',
			    'TFAutoModelForCausalLM',
			    'TFAutoModelForImageClassification',
			    'TFAutoModelForMaskedImageModeling',
			    'TFAutoModelForMaskedLM',
			    'TFAutoModelForMaskGeneration',
			    'TFAutoModelForMultipleChoice',
			    'TFAutoModelForNextSentencePrediction',
			    'TFAutoModelForPreTraining',
			    'TFAutoModelForDocumentQuestionAnswering',
			    'TFAutoModelForQuestionAnswering',
			    'TFAutoModelForSemanticSegmentation',
			    'TFAutoModelForSeq2SeqLM',
			    'TFAutoModelForSequenceClassification',
			    'TFAutoModelForSpeechSeq2Seq',
			    'TFAutoModelForTableQuestionAnswering',
			    'TFAutoModelForTextEncoding',
			    'TFAutoModelForTokenClassification',
			    'TFAutoModelForVision2Seq',
			    'TFAutoModelForZeroShotImageClassification',
			    'TFAutoModelWithLMHead',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase		:						Any =					[
			    'FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
			    'FLAX_MODEL_FOR_CAUSAL_LM_MAPPING',
			    'FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
			    'FLAX_MODEL_FOR_MASKED_LM_MAPPING',
			    'FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
			    'FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
			    'FLAX_MODEL_FOR_PRETRAINING_MAPPING',
			    'FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING',
			    'FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
			    'FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
			    'FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
			    'FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
			    'FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING',
			    'FLAX_MODEL_MAPPING',
			    'FlaxAutoModel',
			    'FlaxAutoModelForCausalLM',
			    'FlaxAutoModelForImageClassification',
			    'FlaxAutoModelForMaskedLM',
			    'FlaxAutoModelForMultipleChoice',
			    'FlaxAutoModelForNextSentencePrediction',
			    'FlaxAutoModelForPreTraining',
			    'FlaxAutoModelForQuestionAnswering',
			    'FlaxAutoModelForSeq2SeqLM',
			    'FlaxAutoModelForSequenceClassification',
			    'FlaxAutoModelForSpeechSeq2Seq',
			    'FlaxAutoModelForTokenClassification',
			    'FlaxAutoModelForVision2Seq',
			]


if TYPE_CHECKING:
			from .auto_factory import get_values
			from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
			from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
			from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
			from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
			from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_auto import (
						    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
						    MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
						    MODEL_FOR_AUDIO_XVECTOR_MAPPING,
						    MODEL_FOR_BACKBONE_MAPPING,
						    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
						    MODEL_FOR_CAUSAL_LM_MAPPING,
						    MODEL_FOR_CTC_MAPPING,
						    MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
						    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
						    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
						    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
						    MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
						    MODEL_FOR_MASK_GENERATION_MAPPING,
						    MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
						    MODEL_FOR_MASKED_LM_MAPPING,
						    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
						    MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
						    MODEL_FOR_OBJECT_DETECTION_MAPPING,
						    MODEL_FOR_PRETRAINING_MAPPING,
						    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
						    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
						    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
						    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
						    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
						    MODEL_FOR_TEXT_ENCODING_MAPPING,
						    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
						    MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
						    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
						    MODEL_FOR_VISION_2_SEQ_MAPPING,
						    MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
						    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
						    MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
						    MODEL_MAPPING,
						    MODEL_WITH_LM_HEAD_MAPPING,
						    AutoBackbone,
						    AutoModel,
						    AutoModelForAudioClassification,
						    AutoModelForAudioFrameClassification,
						    AutoModelForAudioXVector,
						    AutoModelForCausalLM,
						    AutoModelForCTC,
						    AutoModelForDepthEstimation,
						    AutoModelForDocumentQuestionAnswering,
						    AutoModelForImageClassification,
						    AutoModelForImageSegmentation,
						    AutoModelForInstanceSegmentation,
						    AutoModelForMaskedImageModeling,
						    AutoModelForMaskedLM,
						    AutoModelForMaskGeneration,
						    AutoModelForMultipleChoice,
						    AutoModelForNextSentencePrediction,
						    AutoModelForObjectDetection,
						    AutoModelForPreTraining,
						    AutoModelForQuestionAnswering,
						    AutoModelForSemanticSegmentation,
						    AutoModelForSeqaSeqLM,
						    AutoModelForSequenceClassification,
						    AutoModelForSpeechSeqaSeq,
						    AutoModelForTableQuestionAnswering,
						    AutoModelForTextEncoding,
						    AutoModelForTokenClassification,
						    AutoModelForUniversalSegmentation,
						    AutoModelForVideoClassification,
						    AutoModelForVisionaSeq,
						    AutoModelForVisualQuestionAnswering,
						    AutoModelForZeroShotImageClassification,
						    AutoModelForZeroShotObjectDetection,
						    AutoModelWithLMHead,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_auto import (
						    TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
						    TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
						    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_MASK_GENERATION_MAPPING,
						    TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
						    TF_MODEL_FOR_MASKED_LM_MAPPING,
						    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
						    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
						    TF_MODEL_FOR_PRETRAINING_MAPPING,
						    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
						    TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
						    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
						    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
						    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
						    TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
						    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
						    TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
						    TF_MODEL_MAPPING,
						    TF_MODEL_WITH_LM_HEAD_MAPPING,
						    TFAutoModel,
						    TFAutoModelForAudioClassification,
						    TFAutoModelForCausalLM,
						    TFAutoModelForDocumentQuestionAnswering,
						    TFAutoModelForImageClassification,
						    TFAutoModelForMaskedImageModeling,
						    TFAutoModelForMaskedLM,
						    TFAutoModelForMaskGeneration,
						    TFAutoModelForMultipleChoice,
						    TFAutoModelForNextSentencePrediction,
						    TFAutoModelForPreTraining,
						    TFAutoModelForQuestionAnswering,
						    TFAutoModelForSemanticSegmentation,
						    TFAutoModelForSeqaSeqLM,
						    TFAutoModelForSequenceClassification,
						    TFAutoModelForSpeechSeqaSeq,
						    TFAutoModelForTableQuestionAnswering,
						    TFAutoModelForTextEncoding,
						    TFAutoModelForTokenClassification,
						    TFAutoModelForVisionaSeq,
						    TFAutoModelForZeroShotImageClassification,
						    TFAutoModelWithLMHead,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_auto import (
						    FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
						    FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
						    FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
						    FLAX_MODEL_FOR_MASKED_LM_MAPPING,
						    FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
						    FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
						    FLAX_MODEL_FOR_PRETRAINING_MAPPING,
						    FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
						    FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
						    FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
						    FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
						    FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
						    FLAX_MODEL_MAPPING,
						    FlaxAutoModel,
						    FlaxAutoModelForCausalLM,
						    FlaxAutoModelForImageClassification,
						    FlaxAutoModelForMaskedLM,
						    FlaxAutoModelForMultipleChoice,
						    FlaxAutoModelForNextSentencePrediction,
						    FlaxAutoModelForPreTraining,
						    FlaxAutoModelForQuestionAnswering,
						    FlaxAutoModelForSeqaSeqLM,
						    FlaxAutoModelForSequenceClassification,
						    FlaxAutoModelForSpeechSeqaSeq,
						    FlaxAutoModelForTokenClassification,
						    FlaxAutoModelForVisionaSeq,
						)

else:
			import sys

			lowercase		:						Optional[int] =					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


"""simple docstring"""
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class    _snake_case  (						_lowercase ,				_lowercase ,				_lowercase       ):






							@register_to_config
							def __init__(					self						:			str						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			float						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			str						, snake_case__						:			bool = False						, ):






												"""simple docstring"""



												super().__init__()

												__a:						Optional[Any]		    =					nn.Linear(snake_case__						, snake_case__						, bias=snake_case__    )

												__a:						Optional[Any]		    =					nn.Embedding(snake_case__						, snake_case__    )
												__a:						Any		    =					False

												__a:						Any		    =					nn.Dropout(p=snake_case__    )

												__a:						List[str]		    =					TaConfig(
												    d_model=snake_case__						, num_heads=snake_case__						, d_kv=snake_case__						, d_ff=snake_case__						, feed_forward_proj=snake_case__						, dropout_rate=snake_case__						, is_decoder=snake_case__						, is_encoder_decoder=snake_case__						, )
												__a:						Optional[int]		    =					nn.ModuleList()
												for lyr_num in range(snake_case__    ):
																	__a:						Union[str, Any]		    =					TaBlock(snake_case__    )
																	self.encoders.append(snake_case__    )

												__a:						List[str]		    =					TaLayerNorm(snake_case__    )
												__a:						Optional[int]		    =					nn.Dropout(p=snake_case__    )






							def 							__lowercase  (					self						:			str						, snake_case__						:			Optional[Any]						, snake_case__						:			int    ):






												"""simple docstring"""



												__a:						str		    =					self.input_proj(snake_case__    )

												# terminal relative positional encodings
												__a:						Optional[int]		    =					encoder_inputs.shape[1]
												__a:						List[Any]		    =					torch.arange(snake_case__						, device=encoder_inputs.device    )

												__a:						Union[str, Any]		    =					encoder_inputs_mask.sum(-1    )
												__a:						str		    =					torch.roll(input_positions.unsqueeze(0    )						, tuple(seq_lens.tolist()    )						, dims=0    )
												x += self.position_encoding(snake_case__    )

												__a:						Any		    =					self.dropout_pre(snake_case__    )

												# inverted the attention mask
												__a:						Any		    =					encoder_inputs.size()
												__a:						str		    =					self.get_extended_attention_mask(snake_case__						, snake_case__    )

												for lyr in self.encoders:
																	__a:						Any		    =					lyr(snake_case__						, snake_case__    )[0]
												__a:						int		    =					self.layer_norm(snake_case__    )

												return self.dropout_post(snake_case__    ), encoder_inputs_mask
