






'''simple docstring'''





import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class        __snake_case(					__UpperCamelCase     ,       __UpperCamelCase     ,       __UpperCamelCase      ):



 @register_to_config
 def __init__(	self   :   Dict ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   float ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   bool = False ,		):

    """simple docstring"""






    super().__init__()

    lowercase__						:	Union[str, Any]         = nn.Linear(A_ ,		A_ ,		bias=A_	)

    lowercase__						:	Any         = nn.Embedding(A_ ,		A_	)
    lowercase__						:	Tuple         = False

    lowercase__						:	Tuple         = nn.Dropout(p=A_	)

    lowercase__						:	str         = TaConfig(
        d_model=A_ ,		num_heads=A_ ,		d_kv=A_ ,		d_ff=A_ ,		feed_forward_proj=A_ ,		dropout_rate=A_ ,		is_decoder=A_ ,		is_encoder_decoder=A_ ,		)
    lowercase__						:	Tuple         = nn.ModuleList()
    for lyr_num in range(A_	):
       lowercase__						:	str         = TaBlock(A_	)
       self.encoders.append(A_	)

    lowercase__						:	Tuple         = TaLayerNorm(A_	)
    lowercase__						:	List[Any]         = nn.Dropout(p=A_	)







 def 						UpperCAmelCase    (	self   :   str ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[Any]	):

    """simple docstring"""






    lowercase__						:	int         = self.input_proj(A_	)

    # terminal relative positional encodings
    lowercase__						:	Optional[int]         = encoder_inputs.shape[1]
    lowercase__						:	Tuple         = torch.arange(A_ ,		device=encoder_inputs.device	)

    lowercase__						:	Dict         = encoder_inputs_mask.sum(-1	)
    lowercase__						:	Dict         = torch.roll(input_positions.unsqueeze(0	) ,		tuple(seq_lens.tolist()	) ,		dims=0	)
    x += self.position_encoding(A_	)

    lowercase__						:	Dict         = self.dropout_pre(A_	)

    # inverted the attention mask
    lowercase__						:	Optional[Any]         = encoder_inputs.size()
    lowercase__						:	Dict         = self.get_extended_attention_mask(A_ ,		A_	)

    for lyr in self.encoders:
       lowercase__						:	List[str]         = lyr(A_ ,		A_	)[0]
    lowercase__						:	Tuple         = self.layer_norm(A_	)

    return self.dropout_post(A_	), encoder_inputs_mask










a    =       10**8
def       a__						(  A					):
   if n % 10 == 0:
      return False
   lowercase__						:	Optional[Any]         = str(A					)
   return s == s[::-1]
def       a__						(  ):
   lowercase__						:	Union[str, Any]         = set()
   lowercase__						:	Dict         = 1
   lowercase__						:	List[str]         = 5
   while sum_squares < LIMIT:
      lowercase__						:	Tuple         = first_square + 1
      while sum_squares < LIMIT:
         if is_palindrome(A					):
            answer.add(A					)
         last_square += 1
         sum_squares += last_square**2
      first_square += 1
      lowercase__						:	Any         = first_square**2 + (first_square + 1) ** 2

   return sum(A					)


if __name__ == "__main__":
 print(solution())


