

"""simple docstring"""





import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a				   =							logging.get_logger(__name__)

_a				   =							{
    'source_spm': 'source.spm',
    'target_spm': 'target.spm',
    'vocab': 'vocab.json',
    'target_vocab_file': 'target_vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
}

_a				   =							{
    'source_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm'
    },
    'target_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm'
    },
    'vocab': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json'
    },
    'tokenizer_config_file': {
        'Helsinki-NLP/opus-mt-en-de': (
            'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json'
        )
    },
}

_a				   =							{'Helsinki-NLP/opus-mt-en-de': 5_12}
_a				   =							{}




class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 __UpperCAmelCase			:      Optional[int]     	=			VOCAB_FILES_NAMES
 __UpperCAmelCase			:      Any     	=			PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase			:      Dict     	=			PRETRAINED_INIT_CONFIGURATION
 __UpperCAmelCase			:      Tuple     	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase			:      Tuple     	=			["input_ids", "attention_mask"]
 __UpperCAmelCase			:      List[Any]     	=			re.compile(">>.+<<"     )  # type: re.Pattern


 def __init__(    self							:    str,      UpperCAmelCase__							:    int,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    List[Any],      UpperCAmelCase__							:    str=None,      UpperCAmelCase__							:    int=None,      UpperCAmelCase__							:    List[str]=None,      UpperCAmelCase__							:    List[Any]="<unk>",      UpperCAmelCase__							:    str="</s>",      UpperCAmelCase__							:    Any="<pad>",      UpperCAmelCase__							:    List[Any]=5_1_2,      UpperCAmelCase__							:    Optional[Dict[str, Any]] = None,      UpperCAmelCase__							:    Tuple=False,      **UpperCAmelCase__							:    Union[str, Any],      ):
    __lowercase			=		{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
        source_lang=UpperCAmelCase__,      target_lang=UpperCAmelCase__,      unk_token=UpperCAmelCase__,      eos_token=UpperCAmelCase__,      pad_token=UpperCAmelCase__,      model_max_length=UpperCAmelCase__,      sp_model_kwargs=self.sp_model_kwargs,      target_vocab_file=UpperCAmelCase__,      separate_vocabs=UpperCAmelCase__,      **UpperCAmelCase__,      )
    assert Path(UpperCAmelCase__			).exists(), F"""cannot find spm source {source_spm}"""

    __lowercase			=		separate_vocabs
    __lowercase			=		load_json(UpperCAmelCase__			)
    if self.unk_token not in self.encoder:
       raise KeyError("<unk> token must be in vocab"			)
    assert self.pad_token in self.encoder

    if separate_vocabs:
       __lowercase			=		load_json(UpperCAmelCase__			)
       __lowercase			=		{v: k for k, v in self.target_encoder.items()}
       __lowercase			=		[]
    else:
       __lowercase			=		{v: k for k, v in self.encoder.items()}
       __lowercase			=		[k for k in self.encoder if k.startswith(">>"			) and k.endswith("<<"			)]

    __lowercase			=		source_lang
    __lowercase			=		target_lang
    __lowercase			=		[source_spm, target_spm]

    # load SentencePiece model for pre-processing
    __lowercase			=		load_spm(UpperCAmelCase__,      self.sp_model_kwargs			)
    __lowercase			=		load_spm(UpperCAmelCase__,      self.sp_model_kwargs			)
    __lowercase			=		self.spm_source
    __lowercase			=		self.encoder

    # Multilingual target side: default to using first supported language code.

    self._setup_normalizer()


 def       _lowercase				(    self							:    str			):
    try:
       from sacremoses import MosesPunctNormalizer

       __lowercase			=		MosesPunctNormalizer(self.source_lang			).normalize
    except (ImportError, FileNotFoundError):
       warnings.warn("Recommended: pip install sacremoses."			)
       __lowercase			=		lambda UpperCAmelCase__			: x


 def       _lowercase				(    self							:    Tuple,      UpperCAmelCase__							:    str			):
    return self.punc_normalizer(UpperCAmelCase__			) if x else ""


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    str			):
    return self.current_encoder.get(UpperCAmelCase__,      self.current_encoder[self.unk_token]			)


 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    str			):
    __lowercase			=		self.language_code_re.match(UpperCAmelCase__			)
    __lowercase			=		[match.group(0			)] if match else []
    return code, self.language_code_re.sub("",      UpperCAmelCase__			)


 def       _lowercase				(    self							:    List[str],      UpperCAmelCase__							:    str			):
    __lowercase	,__lowercase			=		self.remove_language_code(UpperCAmelCase__			)
    __lowercase			=		self.current_spm.encode(UpperCAmelCase__,      out_type=UpperCAmelCase__			)
    return code + pieces


 def       _lowercase				(    self							:    Dict,      UpperCAmelCase__							:    int			):
    return self.decoder.get(UpperCAmelCase__,      self.unk_token			)


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    str,      **UpperCAmelCase__							:    int			):
    return super().batch_decode(UpperCAmelCase__,      **UpperCAmelCase__			)


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    Tuple,      **UpperCAmelCase__							:    Dict			):
    return super().decode(UpperCAmelCase__,      **UpperCAmelCase__			)


 def       _lowercase				(    self							:    Optional[int],      UpperCAmelCase__							:    List[str]			):
    __lowercase			=		self.spm_source if self._decode_use_source_tokenizer else self.spm_target
    __lowercase			=		[]
    __lowercase			=		""
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          out_string += sp_model.decode_pieces(UpperCAmelCase__			) + token + " "
          __lowercase			=		[]
       else:
          current_sub_tokens.append(UpperCAmelCase__			)
    out_string += sp_model.decode_pieces(UpperCAmelCase__			)
    return out_string.strip()


 def       _lowercase				(    self							:    Optional[int],      UpperCAmelCase__							:    Any,      UpperCAmelCase__							:    str=None			):
    if token_ids_a is None:
       return token_ids_a + [self.eos_token_id]
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return token_ids_a + token_ids_a + [self.eos_token_id]


 def       _lowercase				(    self							:    Optional[Any]			):
    __lowercase			=		self.spm_source
    __lowercase			=		self.encoder


 def       _lowercase				(    self							:    Optional[int]			):
    __lowercase			=		self.spm_target
    if self.separate_vocabs:
       __lowercase			=		self.target_encoder


 @property
 def       _lowercase				(    self							:    Optional[int]			):
    return len(self.encoder			)


 def       _lowercase				(    self							:    Optional[int],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Optional[str] = None			):
    if not os.path.isdir(UpperCAmelCase__			):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
       return
    __lowercase			=		[]

    if self.separate_vocabs:
       __lowercase			=		os.path.join(
           UpperCAmelCase__,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"],      )
       __lowercase			=		os.path.join(
           UpperCAmelCase__,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["target_vocab_file"],      )
       save_json(self.encoder,      UpperCAmelCase__			)
       save_json(self.target_encoder,      UpperCAmelCase__			)
       saved_files.append(UpperCAmelCase__			)
       saved_files.append(UpperCAmelCase__			)
    else:
       __lowercase			=		os.path.join(
           UpperCAmelCase__,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"]			)
       save_json(self.encoder,      UpperCAmelCase__			)
       saved_files.append(UpperCAmelCase__			)

    for spm_save_filename, spm_orig_path, spm_model in zip(
        [VOCAB_FILES_NAMES["source_spm"], VOCAB_FILES_NAMES["target_spm"]],      self.spm_files,      [self.spm_source, self.spm_target],      ):
       __lowercase			=		os.path.join(
           UpperCAmelCase__,      (filename_prefix + "-" if filename_prefix else "") + spm_save_filename			)
       if os.path.abspath(UpperCAmelCase__			) != os.path.abspath(UpperCAmelCase__			) and os.path.isfile(UpperCAmelCase__			):
          copyfile(UpperCAmelCase__,      UpperCAmelCase__			)
          saved_files.append(UpperCAmelCase__			)
       elif not os.path.isfile(UpperCAmelCase__			):
          with open(UpperCAmelCase__,      "wb"			) as fi:
             __lowercase			=		spm_model.serialized_model_proto()
             fi.write(UpperCAmelCase__			)
          saved_files.append(UpperCAmelCase__			)

    return tuple(UpperCAmelCase__			)


 def       _lowercase				(    self							:    int			):
    return self.get_src_vocab()


 def       _lowercase				(    self							:    List[str]			):
    return dict(self.encoder,      **self.added_tokens_encoder			)


 def       _lowercase				(    self							:    Tuple			):
    return dict(self.target_encoder,      **self.added_tokens_decoder			)


 def __getstate__(    self							:    int			):
    __lowercase			=		self.__dict__.copy()
    state.update(
        {k: None for k in ["spm_source", "spm_target", "current_spm", "punc_normalizer", "target_vocab_file"]}			)
    return state


 def __setstate__(    self							:    str,      UpperCAmelCase__							:    Dict			):
    __lowercase			=		d

    # for backward compatibility
    if not hasattr(self,      "sp_model_kwargs"			):
       __lowercase			=		{}

    __lowercase	,__lowercase			=		(load_spm(UpperCAmelCase__,      self.sp_model_kwargs			) for f in self.spm_files)
    __lowercase			=		self.spm_source
    self._setup_normalizer()


 def       _lowercase				(    self							:    str,      *UpperCAmelCase__							:    int,      **UpperCAmelCase__							:    List[Any]			):
    return 1


 def       _lowercase				(    self							:    Dict,      UpperCAmelCase__							:    Optional[Any]			):
    __lowercase			=		set(self.all_special_ids			)  # call it once instead of inside list comp
    all_special_ids.remove(self.unk_token_id			)  # <unk> is only sometimes special
    return [1 if x in all_special_ids else 0 for x in seq]



 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    List,      UpperCAmelCase__							:    Optional[List] = None,      UpperCAmelCase__							:    bool = False			):
    if already_has_special_tokens:
       return self._special_token_mask(UpperCAmelCase__			)
    elif token_ids_a is None:
       return self._special_token_mask(UpperCAmelCase__			) + [1]
    else:
       return self._special_token_mask(token_ids_a + token_ids_a			) + [1]







def 						_A						(						UpperCamelCase_  : str,    UpperCamelCase_  : Dict[str, Any])			->		sentencepiece.SentencePieceProcessor:
   '''simple docstring'''


   __lowercase			=		sentencepiece.SentencePieceProcessor(**UpperCamelCase_)
   spm.Load(UpperCamelCase_)
   return spm




def 						_A						(						UpperCamelCase_  : Any,    UpperCamelCase_  : str)			->		None:
   '''simple docstring'''


   with open(UpperCamelCase_,    "w") as f:
      json.dump(UpperCamelCase_,    UpperCamelCase_,    indent=2)




def 						_A						(						UpperCamelCase_  : str)			->		Union[Dict, List]:
   '''simple docstring'''


   with open(UpperCamelCase_,    "r") as f:
      return json.load(UpperCamelCase_)







"""simple docstring"""





import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def 						_A						(						UpperCamelCase_  : int,    UpperCamelCase_  : Union[str, Any],    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Dict,    UpperCamelCase_  : int = 4096)			->		Optional[Any]:
   '''simple docstring'''


   __lowercase	,__lowercase	,__lowercase			=		key.shape[-3:]
   __lowercase			=		value.shape[-1]
   __lowercase			=		min(UpperCamelCase_,    UpperCamelCase_)
   __lowercase			=		query / jnp.sqrt(UpperCamelCase_)

   @functools.partial(jax.checkpoint,    prevent_cse=UpperCamelCase_)
   def summarize_chunk(UpperCamelCase_  : int,    UpperCamelCase_  : Union[str, Any],    UpperCamelCase_  : Any):
      __lowercase			=		jnp.einsum("...qhd,...khd->...qhk",    UpperCamelCase_,    UpperCamelCase_,    precision=UpperCamelCase_)

      __lowercase			=		jnp.max(UpperCamelCase_,    axis=-1,    keepdims=UpperCamelCase_)
      __lowercase			=		jax.lax.stop_gradient(UpperCamelCase_)
      __lowercase			=		jnp.exp(attn_weights - max_score)

      __lowercase			=		jnp.einsum("...vhf,...qhv->...qhf",    UpperCamelCase_,    UpperCamelCase_,    precision=UpperCamelCase_)
      __lowercase			=		jnp.einsum("...qhk->...qh",    UpperCamelCase_)

      return (exp_values, exp_weights.sum(axis=-1), max_score)

   def chunk_scanner(UpperCamelCase_  : int):
      # julienne key array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0],    slice_sizes=list(key.shape[:-3]) + [key_chunk_size, num_heads, k_features],    )

      # julienne value array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0],    slice_sizes=list(value.shape[:-3]) + [key_chunk_size, num_heads, v_features],    )

      return summarize_chunk(UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_)

   __lowercase	,__lowercase	,__lowercase			=		jax.lax.map(f=UpperCamelCase_,    xs=jnp.arange(0,    UpperCamelCase_,    UpperCamelCase_))

   __lowercase			=		jnp.max(UpperCamelCase_,    axis=0,    keepdims=UpperCamelCase_)
   __lowercase			=		jnp.exp(chunk_max - global_max)

   chunk_values *= jnp.expand_dims(UpperCamelCase_,    axis=-1)
   chunk_weights *= max_diffs

   __lowercase			=		chunk_values.sum(axis=0)
   __lowercase			=		jnp.expand_dims(UpperCamelCase_,    -1).sum(axis=0)

   return all_values / all_weights




def 						_A						(						UpperCamelCase_  : Optional[Any],    UpperCamelCase_  : Optional[int],    UpperCamelCase_  : List[str],    UpperCamelCase_  : Union[str, Any]=jax.lax.Precision.HIGHEST,    UpperCamelCase_  : int = 1024,    UpperCamelCase_  : int = 4096)			->		str:
   '''simple docstring'''


   __lowercase	,__lowercase	,__lowercase			=		query.shape[-3:]

   def chunk_scanner(UpperCamelCase_  : List[str],    UpperCamelCase_  : str):
      # julienne query array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0],    slice_sizes=list(query.shape[:-3]) + [min(UpperCamelCase_,    UpperCamelCase_), num_heads, q_features],    )

      return (
          chunk_idx + query_chunk_size,  # unused ignore it
          _query_chunk_attention(
              query=UpperCamelCase_,    key=UpperCamelCase_,    value=UpperCamelCase_,    precision=UpperCamelCase_,    key_chunk_size=UpperCamelCase_),
      )

   __lowercase	,__lowercase			=		jax.lax.scan(
       f=UpperCamelCase_,    init=0,    xs=UpperCamelCase_,    length=math.ceil(num_q / query_chunk_size))

   return jnp.concatenate(UpperCamelCase_,    axis=-3)  # fuse the chunked result back




class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int     	=			8
 __UpperCAmelCase			:      int     	=			6_4
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    List[Any]			):
    __lowercase			=		self.dim_head * self.heads
    __lowercase			=		self.dim_head**-0.5

    # Weights were exported with old names {to_q, to_k, to_v, to_out}
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_q"			)
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_k"			)
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_v"			)

    __lowercase			=		nn.Dense(self.query_dim,      dtype=self.dtype,      name="to_out_0"			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)


 def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    Union[str, Any]			):
    __lowercase	,__lowercase	,__lowercase			=		tensor.shape
    __lowercase			=		self.heads
    __lowercase			=		tensor.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      dim // head_size			)
    __lowercase			=		jnp.transpose(UpperCAmelCase__,      (0, 2, 1, 3)			)
    __lowercase			=		tensor.reshape(batch_size * head_size,      UpperCAmelCase__,      dim // head_size			)
    return tensor


 def       _lowercase				(    self							:    Union[str, Any],      UpperCAmelCase__							:    int			):
    __lowercase	,__lowercase	,__lowercase			=		tensor.shape
    __lowercase			=		self.heads
    __lowercase			=		tensor.reshape(batch_size // head_size,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
    __lowercase			=		jnp.transpose(UpperCAmelCase__,      (0, 2, 1, 3)			)
    __lowercase			=		tensor.reshape(batch_size // head_size,      UpperCAmelCase__,      dim * head_size			)
    return tensor



 def __call__(    self							:    List[Any],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Union[str, Any]=None,      UpperCAmelCase__							:    Optional[Any]=True			):
    __lowercase			=		hidden_states if context is None else context

    __lowercase			=		self.query(UpperCAmelCase__			)
    __lowercase			=		self.key(UpperCAmelCase__			)
    __lowercase			=		self.value(UpperCAmelCase__			)

    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)
    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)
    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)

    if self.use_memory_efficient_attention:
       __lowercase			=		query_states.transpose(1,      0,      2			)
       __lowercase			=		key_states.transpose(1,      0,      2			)
       __lowercase			=		value_states.transpose(1,      0,      2			)

       # this if statement create a chunk size for each layer of the unet
       # the chunk size is equal to the query_length dimension of the deepest layer of the unet

       __lowercase			=		query_states.shape[-3]
       if flatten_latent_dim % 6_4 == 0:
          __lowercase			=		int(flatten_latent_dim / 6_4			)
       elif flatten_latent_dim % 1_6 == 0:
          __lowercase			=		int(flatten_latent_dim / 1_6			)
       elif flatten_latent_dim % 4 == 0:
          __lowercase			=		int(flatten_latent_dim / 4			)
       else:
          __lowercase			=		int(UpperCAmelCase__			)

       __lowercase			=		jax_memory_efficient_attention(
           UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      query_chunk_size=UpperCAmelCase__,      key_chunk_size=4_0_9_6 * 4			)

       __lowercase			=		hidden_states.transpose(1,      0,      2			)
    else:
       # compute attentions
       __lowercase			=		jnp.einsum("b i d, b j d->b i j",      UpperCAmelCase__,      UpperCAmelCase__			)
       __lowercase			=		attention_scores * self.scale
       __lowercase			=		nn.softmax(UpperCAmelCase__,      axis=2			)

       # attend to values
       __lowercase			=		jnp.einsum("b i j, b j d -> b i d",      UpperCAmelCase__,      UpperCAmelCase__			)

    __lowercase			=		self.reshape_batch_dim_to_heads(UpperCAmelCase__			)
    __lowercase			=		self.proj_attn(UpperCAmelCase__			)
    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa
 __UpperCAmelCase			:      bool     	=			False


 def       _lowercase				(    self							:    Tuple			):
    # self attention (or cross_attention if only_cross_attention is True)
    __lowercase			=		FlaxAttention(
        self.dim,      self.n_heads,      self.d_head,      self.dropout,      self.use_memory_efficient_attention,      dtype=self.dtype			)
    # cross attention
    __lowercase			=		FlaxAttention(
        self.dim,      self.n_heads,      self.d_head,      self.dropout,      self.use_memory_efficient_attention,      dtype=self.dtype			)
    __lowercase			=		FlaxFeedForward(dim=self.dim,      dropout=self.dropout,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    str,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    Dict=True			):
    # self attention
    __lowercase			=		hidden_states
    if self.only_cross_attention:
       __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    else:
       __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    # cross attention
    __lowercase			=		hidden_states
    __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    # feed forward
    __lowercase			=		hidden_states
    __lowercase			=		self.ff(self.norma(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int     	=			1
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa
 __UpperCAmelCase			:      bool     	=			False


 def       _lowercase				(    self							:    Dict			):
    __lowercase			=		nn.GroupNorm(num_groups=3_2,      epsilon=1E-5			)

    __lowercase			=		self.n_heads * self.d_head
    if self.use_linear_projection:
       __lowercase			=		nn.Dense(UpperCAmelCase__,      dtype=self.dtype			)
    else:
       __lowercase			=		nn.Conv(
           UpperCAmelCase__,      kernel_size=(1, 1),      strides=(1, 1),      padding="VALID",      dtype=self.dtype,      )

    __lowercase			=		[
        FlaxBasicTransformerBlock(
            UpperCAmelCase__,      self.n_heads,      self.d_head,      dropout=self.dropout,      only_cross_attention=self.only_cross_attention,      dtype=self.dtype,      use_memory_efficient_attention=self.use_memory_efficient_attention,      )
        for _ in range(self.depth			)
    ]

    if self.use_linear_projection:
       __lowercase			=		nn.Dense(UpperCAmelCase__,      dtype=self.dtype			)
    else:
       __lowercase			=		nn.Conv(
           UpperCAmelCase__,      kernel_size=(1, 1),      strides=(1, 1),      padding="VALID",      dtype=self.dtype,      )

    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    Dict,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    List[str]=True			):
    __lowercase	,__lowercase	,__lowercase	,__lowercase			=		hidden_states.shape
    __lowercase			=		hidden_states
    __lowercase			=		self.norm(UpperCAmelCase__			)
    if self.use_linear_projection:
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      height * width,      UpperCAmelCase__			)
       __lowercase			=		self.proj_in(UpperCAmelCase__			)
    else:
       __lowercase			=		self.proj_in(UpperCAmelCase__			)
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      height * width,      UpperCAmelCase__			)

    for transformer_block in self.transformer_blocks:
       __lowercase			=		transformer_block(UpperCAmelCase__,      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)

    if self.use_linear_projection:
       __lowercase			=		self.proj_out(UpperCAmelCase__			)
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
    else:
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
       __lowercase			=		self.proj_out(UpperCAmelCase__			)

    __lowercase			=		hidden_states + residual
    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    List[str]			):
    # The second linear layer needs to be called
    # net_2 for now to match the index of the Sequential layer
    __lowercase			=		FlaxGEGLU(self.dim,      self.dropout,      self.dtype			)
    __lowercase			=		nn.Dense(self.dim,      dtype=self.dtype			)



 def __call__(    self							:    int,      UpperCAmelCase__							:    List[str],      UpperCAmelCase__							:    Union[str, Any]=True			):
    __lowercase			=		self.net_a(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    __lowercase			=		self.net_a(UpperCAmelCase__			)
    return hidden_states







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    int			):
    __lowercase			=		self.dim * 4
    __lowercase			=		nn.Dense(inner_dim * 2,      dtype=self.dtype			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    Optional[Any],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    int=True			):
    __lowercase			=		self.proj(UpperCAmelCase__			)
    __lowercase	,__lowercase			=		jnp.split(UpperCAmelCase__,      2,      axis=2			)
    return self.dropout_layer(hidden_linear * nn.gelu(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)




