





'''simple docstring'''






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class 				UpperCAmelCase  (    snake_case_							,	snake_case_							,	snake_case_     ):




      @register_to_config
      def __init__(    self    :				Dict    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				float    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				str    ,    __snake_case    :				bool = False    ,    )       ->   Dict:
            super().__init__()

            _lowerCAmelCase							  =							nn.Linear(__snake_case    ,    __snake_case    ,    bias=__snake_case						)

            _lowerCAmelCase							  =							nn.Embedding(__snake_case    ,    __snake_case						)
            _lowerCAmelCase							  =							False

            _lowerCAmelCase							  =							nn.Dropout(p=__snake_case						)

            _lowerCAmelCase							  =							TaConfig(
                d_model=__snake_case    ,    num_heads=__snake_case    ,    d_kv=__snake_case    ,    d_ff=__snake_case    ,    feed_forward_proj=__snake_case    ,    dropout_rate=__snake_case    ,    is_decoder=__snake_case    ,    is_encoder_decoder=__snake_case    ,    )
            _lowerCAmelCase							  =							nn.ModuleList()
            for lyr_num in range(__snake_case						):
                  _lowerCAmelCase							  =							TaBlock(__snake_case						)
                  self.encoders.append(__snake_case						)

            _lowerCAmelCase							  =							TaLayerNorm(__snake_case						)
            _lowerCAmelCase							  =							nn.Dropout(p=__snake_case						)




      def        lowercase__					(    self    :				Union[str, Any]    ,    __snake_case    :				Optional[int]    ,    __snake_case    :				List[Any]						)       ->   Optional[Any]:
            _lowerCAmelCase							  =							self.input_proj(__snake_case						)

            # terminal relative positional encodings
            _lowerCAmelCase							  =							encoder_inputs.shape[1]
            _lowerCAmelCase							  =							torch.arange(__snake_case    ,    device=encoder_inputs.device						)

            _lowerCAmelCase							  =							encoder_inputs_mask.sum(-1						)
            _lowerCAmelCase							  =							torch.roll(input_positions.unsqueeze(0						)    ,    tuple(seq_lens.tolist()						)    ,    dims=0						)
            x += self.position_encoding(__snake_case						)

            _lowerCAmelCase							  =							self.dropout_pre(__snake_case						)

            # inverted the attention mask
            _lowerCAmelCase							  =							encoder_inputs.size()
            _lowerCAmelCase							  =							self.get_extended_attention_mask(__snake_case    ,    __snake_case						)

            for lyr in self.encoders:
                  _lowerCAmelCase							  =							lyr(__snake_case    ,    __snake_case						)[0]
            _lowerCAmelCase							  =							self.layer_norm(__snake_case						)

            return self.dropout_post(__snake_case						), encoder_inputs_mask






'''simple docstring'''






import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig




class 				UpperCAmelCase  (    snake_case_     ):
      _lowercase:    str		  =     CustomConfig




      def __init__(    self    :				Optional[int]    ,    __snake_case    :				Tuple						)       ->   Optional[Any]:
            super().__init__(__snake_case						)
            _lowerCAmelCase							  =							torch.nn.Linear(config.hidden_size    ,    config.hidden_size						)




      def        lowercase__					(    self    :				Dict    ,    __snake_case    :				List[Any]						)       ->   Dict:
            return self.linear(__snake_case						)




      def        lowercase__					(    self    :				List[Any]    ,    __snake_case    :				int						)       ->   Any:
            pass




class 				UpperCAmelCase  (    snake_case_     ):
      _lowercase:    List[str]		  =     NoSuperInitConfig




      def __init__(    self    :				List[str]    ,    __snake_case    :				List[str]						)       ->   int:
            super().__init__(__snake_case						)
            _lowerCAmelCase							  =							torch.nn.Linear(config.attribute    ,    config.attribute						)




      def        lowercase__					(    self    :				List[str]    ,    __snake_case    :				Dict						)       ->   Any:
            return self.linear(__snake_case						)




      def        lowercase__					(    self    :				Optional[int]    ,    __snake_case    :				List[Any]						)       ->   Dict:
            pass






