import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_SCREAMING_SNAKE_CASE:						List[str]    	=       logging.getLogger(__name__)

class 	_SCREAMING_SNAKE_CASE							(     a__							):






		"""simple docstring"""







		def 				_a				(			self    :				Any  ,					__SCREAMING_SNAKE_CASE    :				Optional[Any]  ,					__SCREAMING_SNAKE_CASE    :				Dict  ,					__SCREAMING_SNAKE_CASE    :				Dict=None  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None   ):






					"""simple docstring"""
					if current_layer == 0:
								_snake_case              =      self.embedding_hidden_mapping_in(SCREAMING_SNAKE_CASE_   )
					else:
								_snake_case              =      hidden_states[0]

					_snake_case              =      int(self.config.num_hidden_layers / self.config.num_hidden_groups   )

					# Index of the hidden group
					_snake_case              =      int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)   )

					_snake_case              =      self.albert_layer_groups[group_idx](
					    SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_  ,					head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]  ,					)
					_snake_case              =      layer_group_output[0]

					return (hidden_states,)

@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''      ,     a__      ,     )
class 	_SCREAMING_SNAKE_CASE							(     a__							):






		"""simple docstring"""







		def __init__(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				Any   ):






					"""simple docstring"""
					super().__init__(SCREAMING_SNAKE_CASE_   )

					_snake_case              =      AlbertTransformerWithPabee(SCREAMING_SNAKE_CASE_   )

					self.init_weights()
					_snake_case              =      0
					_snake_case              =      0
					_snake_case              =      0

					_snake_case              =      0



		def 				_a				(			self    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]   ):






					"""simple docstring"""
					_snake_case              =      threshold



		def 				_a				(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				Optional[int]   ):






					"""simple docstring"""
					_snake_case              =      patience



		def 				_a				(			self    :				Any   ):






					"""simple docstring"""
					_snake_case              =      0
					_snake_case              =      0



		def 				_a				(			self    :				Tuple   ):






					"""simple docstring"""
					_snake_case              =      self.inference_layers_num / self.inference_instances_num
					_snake_case              =      (
					    F'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
					    F' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
					)
					print(SCREAMING_SNAKE_CASE_   )







		@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_   )
		def 				_a				(			self    :				Optional[Any]  ,					__SCREAMING_SNAKE_CASE    :				str=None  ,					__SCREAMING_SNAKE_CASE    :				List[Any]=None  ,					__SCREAMING_SNAKE_CASE    :				Any=None  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None  ,					__SCREAMING_SNAKE_CASE    :				Optional[Any]=None  ,					__SCREAMING_SNAKE_CASE    :				Optional[int]=None  ,					__SCREAMING_SNAKE_CASE    :				str=False  ,					):






					"""simple docstring"""

					if input_ids is not None and inputs_embeds is not None:
								raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"   )
					elif input_ids is not None:
								_snake_case              =      input_ids.size()
					elif inputs_embeds is not None:
								_snake_case              =      inputs_embeds.size()[:-1]
					else:
								raise ValueError("You have to specify either input_ids or inputs_embeds"   )

					_snake_case              =      input_ids.device if input_ids is not None else inputs_embeds.device

					if attention_mask is None:
								_snake_case              =      torch.ones(SCREAMING_SNAKE_CASE_  ,					device=SCREAMING_SNAKE_CASE_   )
					if token_type_ids is None:
								_snake_case              =      torch.zeros(SCREAMING_SNAKE_CASE_  ,					dtype=torch.long  ,					device=SCREAMING_SNAKE_CASE_   )

					_snake_case              =      attention_mask.unsqueeze(1   ).unsqueeze(2   )
					_snake_case              =      extended_attention_mask.to(dtype=self.dtype   )  # fp16 compatibility
					_snake_case              =      (1.0 - extended_attention_mask) * -10_000.0
					_snake_case              =      self.get_head_mask(SCREAMING_SNAKE_CASE_  ,					self.config.num_hidden_layers   )

					_snake_case              =      self.embeddings(
					    SCREAMING_SNAKE_CASE_  ,					position_ids=SCREAMING_SNAKE_CASE_  ,					token_type_ids=SCREAMING_SNAKE_CASE_  ,					inputs_embeds=SCREAMING_SNAKE_CASE_   )
					_snake_case              =      embedding_output

					if self.training:
								_snake_case              =      []
								for i in range(self.config.num_hidden_layers   ):
											_snake_case              =      self.encoder.adaptive_forward(
											    SCREAMING_SNAKE_CASE_  ,					current_layer=SCREAMING_SNAKE_CASE_  ,					attention_mask=SCREAMING_SNAKE_CASE_  ,					head_mask=SCREAMING_SNAKE_CASE_  ,					)

											_snake_case              =      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
											_snake_case              =      output_layers[i](output_dropout(SCREAMING_SNAKE_CASE_   )   )
											res.append(SCREAMING_SNAKE_CASE_   )
					elif self.patience == 0:  # Use all layers for inference
								_snake_case              =      self.encoder(SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_  ,					head_mask=SCREAMING_SNAKE_CASE_   )
								_snake_case              =      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
								_snake_case              =      [output_layers[self.config.num_hidden_layers - 1](SCREAMING_SNAKE_CASE_   )]
					else:
								_snake_case              =      0
								_snake_case              =      None
								_snake_case              =      0
								for i in range(self.config.num_hidden_layers   ):
											calculated_layer_num += 1
											_snake_case              =      self.encoder.adaptive_forward(
											    SCREAMING_SNAKE_CASE_  ,					current_layer=SCREAMING_SNAKE_CASE_  ,					attention_mask=SCREAMING_SNAKE_CASE_  ,					head_mask=SCREAMING_SNAKE_CASE_  ,					)

											_snake_case              =      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
											_snake_case              =      output_layers[i](SCREAMING_SNAKE_CASE_   )
											if regression:
														_snake_case              =      logits.detach()
														if patient_result is not None:
																	_snake_case              =      patient_result.detach()
														if (patient_result is not None) and torch.abs(patient_result - labels   ) < self.regression_threshold:
																	patient_counter += 1
														else:
																	_snake_case              =      0
											else:
														_snake_case              =      logits.detach().argmax(dim=1   )
														if patient_result is not None:
																	_snake_case              =      patient_result.detach().argmax(dim=1   )
														if (patient_result is not None) and torch.all(labels.eq(SCREAMING_SNAKE_CASE_   )   ):
																	patient_counter += 1
														else:
																	_snake_case              =      0

											_snake_case              =      logits
											if patient_counter == self.patience:
														break
								_snake_case              =      [patient_result]
								self.inference_layers_num += calculated_layer_num
								self.inference_instances_num += 1

					return res

@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '''      ,     a__      ,     )
class 	_SCREAMING_SNAKE_CASE							(     a__							):






		"""simple docstring"""







		def __init__(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				List[str]   ):






					"""simple docstring"""
					super().__init__(SCREAMING_SNAKE_CASE_   )
					_snake_case              =      config.num_labels

					_snake_case              =      AlbertModelWithPabee(SCREAMING_SNAKE_CASE_   )
					_snake_case              =      nn.Dropout(config.classifier_dropout_prob   )
					_snake_case              =      nn.ModuleList(
					    [nn.Linear(config.hidden_size  ,					self.config.num_labels   ) for _ in range(config.num_hidden_layers   )]   )

					self.init_weights()







		@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_   )
		def 				_a				(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				str=None  ,					__SCREAMING_SNAKE_CASE    :				Optional[Any]=None  ,					__SCREAMING_SNAKE_CASE    :				List[str]=None  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]=None  ,					__SCREAMING_SNAKE_CASE    :				int=None  ,					__SCREAMING_SNAKE_CASE    :				str=None  ,					__SCREAMING_SNAKE_CASE    :				str=None  ,					):






					"""simple docstring"""
					_snake_case              =      self.albert(
					    input_ids=SCREAMING_SNAKE_CASE_  ,					attention_mask=SCREAMING_SNAKE_CASE_  ,					token_type_ids=SCREAMING_SNAKE_CASE_  ,					position_ids=SCREAMING_SNAKE_CASE_  ,					head_mask=SCREAMING_SNAKE_CASE_  ,					inputs_embeds=SCREAMING_SNAKE_CASE_  ,					output_dropout=self.dropout  ,					output_layers=self.classifiers  ,					regression=self.num_labels == 1  ,					)

					_snake_case              =      (logits[-1],)

					if labels is not None:
								_snake_case              =      None
								_snake_case              =      0
								for ix, logits_item in enumerate(SCREAMING_SNAKE_CASE_   ):
											if self.num_labels == 1:
														#  We are doing regression
														_snake_case              =      MSELoss()
														_snake_case              =      loss_fct(logits_item.view(-1   )  ,					labels.view(-1   )   )
											else:
														_snake_case              =      CrossEntropyLoss()
														_snake_case              =      loss_fct(logits_item.view(-1  ,					self.num_labels   )  ,					labels.view(-1   )   )
											if total_loss is None:
														_snake_case              =      loss
											else:
														total_loss += loss * (ix + 1)
											total_weights += ix + 1
								_snake_case              =      (total_loss / total_weights,) + outputs

					return outputs

import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp






def    UpperCAmelCase__(						lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_ = 4096  ):



			"""simple docstring"""





			_snake_case       , _snake_case       , _snake_case              =      key.shape[-3:]
			_snake_case              =      value.shape[-1]
			_snake_case              =      min(lowerCamelCase_							,	lowerCamelCase_  )
			_snake_case              =      query / jnp.sqrt(lowerCamelCase_  )

			@functools.partial(jax.checkpoint							,	prevent_cse=lowerCamelCase_  )
			def summarize_chunk(lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_  ):
						_snake_case              =      jnp.einsum("...qhd,...khd->...qhk"							,	lowerCamelCase_							,	lowerCamelCase_							,	precision=lowerCamelCase_  )

						_snake_case              =      jnp.max(lowerCamelCase_							,	axis=-1							,	keepdims=lowerCamelCase_  )
						_snake_case              =      jax.lax.stop_gradient(lowerCamelCase_  )
						_snake_case              =      jnp.exp(attn_weights - max_score  )

						_snake_case              =      jnp.einsum("...vhf,...qhv->...qhf"							,	lowerCamelCase_							,	lowerCamelCase_							,	precision=lowerCamelCase_  )
						_snake_case              =      jnp.einsum("...qhk->...qh"							,	lowerCamelCase_  )

						return (exp_values, exp_weights.sum(axis=-1  ), max_score)

			def chunk_scanner(lowerCamelCase_  ):
						# julienne key array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,	slice_sizes=list(key.shape[:-3]  ) + [key_chunk_size, num_heads, k_features]							,	)

						# julienne value array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,	slice_sizes=list(value.shape[:-3]  ) + [key_chunk_size, num_heads, v_features]							,	)

						return summarize_chunk(lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_  )

			_snake_case       , _snake_case       , _snake_case              =      jax.lax.map(f=lowerCamelCase_							,	xs=jnp.arange(0							,	lowerCamelCase_							,	lowerCamelCase_  )  )

			_snake_case              =      jnp.max(lowerCamelCase_							,	axis=0							,	keepdims=lowerCamelCase_  )
			_snake_case              =      jnp.exp(chunk_max - global_max  )

			chunk_values *= jnp.expand_dims(lowerCamelCase_							,	axis=-1  )
			chunk_weights *= max_diffs

			_snake_case              =      chunk_values.sum(axis=0  )
			_snake_case              =      jnp.expand_dims(lowerCamelCase_							,	-1  ).sum(axis=0  )

			return all_values / all_weights






def    UpperCAmelCase__(						lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_=jax.lax.Precision.HIGHEST							,	lowerCamelCase_ = 1024							,	lowerCamelCase_ = 4096  ):



			"""simple docstring"""





			_snake_case       , _snake_case       , _snake_case              =      query.shape[-3:]

			def chunk_scanner(lowerCamelCase_							,	lowerCamelCase_  ):
						# julienne query array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,	slice_sizes=list(query.shape[:-3]  ) + [min(lowerCamelCase_							,	lowerCamelCase_  ), num_heads, q_features]							,	)

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=lowerCamelCase_							,	key=lowerCamelCase_							,	value=lowerCamelCase_							,	precision=lowerCamelCase_							,	key_chunk_size=lowerCamelCase_  ),
						)

			_snake_case       , _snake_case              =      jax.lax.scan(
			    f=lowerCamelCase_							,	init=0							,	xs=lowerCamelCase_							,	length=math.ceil(num_q / query_chunk_size  )  )

			return jnp.concatenate(lowerCamelCase_							,	axis=-3  )  # fuse the chunked result back

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      8
		A_						 =      64
		A_						 =      0.0
		A_						 =      False
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					_snake_case              =      self.dim_head * self.heads
					_snake_case              =      self.dim_head**-0.5

					# Weights were exported with old names {to_q, to_k, to_v, to_out}
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_q"   )
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_k"   )
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_v"   )

					_snake_case              =      nn.Dense(self.query_dim  ,					dtype=self.dtype  ,					name="to_out_0"   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )



		def 				_a				(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				Any   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case              =      tensor.shape
					_snake_case              =      self.heads
					_snake_case              =      tensor.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					dim // head_size   )
					_snake_case              =      jnp.transpose(__SCREAMING_SNAKE_CASE  ,					(0, 2, 1, 3)   )
					_snake_case              =      tensor.reshape(batch_size * head_size  ,					__SCREAMING_SNAKE_CASE  ,					dim // head_size   )
					return tensor



		def 				_a				(			self    :				str  ,					__SCREAMING_SNAKE_CASE    :				Tuple   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case              =      tensor.shape
					_snake_case              =      self.heads
					_snake_case              =      tensor.reshape(batch_size // head_size  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					_snake_case              =      jnp.transpose(__SCREAMING_SNAKE_CASE  ,					(0, 2, 1, 3)   )
					_snake_case              =      tensor.reshape(batch_size // head_size  ,					__SCREAMING_SNAKE_CASE  ,					dim * head_size   )
					return tensor







		def __call__(			self    :				int  ,					__SCREAMING_SNAKE_CASE    :				int  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None  ,					__SCREAMING_SNAKE_CASE    :				List[Any]=True   ):






					"""simple docstring"""
					_snake_case              =      hidden_states if context is None else context

					_snake_case              =      self.query(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.key(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.value(__SCREAMING_SNAKE_CASE   )

					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )

					if self.use_memory_efficient_attention:
								_snake_case              =      query_states.transpose(1  ,					0  ,					2   )
								_snake_case              =      key_states.transpose(1  ,					0  ,					2   )
								_snake_case              =      value_states.transpose(1  ,					0  ,					2   )

								# this if statement create a chunk size for each layer of the unet
								# the chunk size is equal to the query_length dimension of the deepest layer of the unet

								_snake_case              =      query_states.shape[-3]
								if flatten_latent_dim % 64 == 0:
											_snake_case              =      int(flatten_latent_dim / 64   )
								elif flatten_latent_dim % 16 == 0:
											_snake_case              =      int(flatten_latent_dim / 16   )
								elif flatten_latent_dim % 4 == 0:
											_snake_case              =      int(flatten_latent_dim / 4   )
								else:
											_snake_case              =      int(__SCREAMING_SNAKE_CASE   )

								_snake_case              =      jax_memory_efficient_attention(
								    __SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					query_chunk_size=__SCREAMING_SNAKE_CASE  ,					key_chunk_size=4_096 * 4   )

								_snake_case              =      hidden_states.transpose(1  ,					0  ,					2   )
					else:
								# compute attentions
								_snake_case              =      jnp.einsum("b i d, b j d->b i j"  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      attention_scores * self.scale
								_snake_case              =      nn.softmax(__SCREAMING_SNAKE_CASE  ,					axis=2   )

								# attend to values
								_snake_case              =      jnp.einsum("b i j, b j d -> b i d"  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )

					_snake_case              =      self.reshape_batch_dim_to_heads(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.proj_attn(__SCREAMING_SNAKE_CASE   )
					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      42
		A_						 =      42
		A_						 =      0.0
		A_						 =      False
		A_						 =      jnp.floataa
		A_						 =      False



		def 				_a				(			self    :				Any   ):






					"""simple docstring"""
					# self attention (or cross_attention if only_cross_attention is True)
					_snake_case              =      FlaxAttention(
					    self.dim  ,					self.n_heads  ,					self.d_head  ,					self.dropout  ,					self.use_memory_efficient_attention  ,					dtype=self.dtype   )
					# cross attention
					_snake_case              =      FlaxAttention(
					    self.dim  ,					self.n_heads  ,					self.d_head  ,					self.dropout  ,					self.use_memory_efficient_attention  ,					dtype=self.dtype   )
					_snake_case              =      FlaxFeedForward(dim=self.dim  ,					dropout=self.dropout  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				List[Any]  ,					__SCREAMING_SNAKE_CASE    :				Any  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				Any=True   ):






					"""simple docstring"""
					# self attention
					_snake_case              =      hidden_states
					if self.only_cross_attention:
								_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					# cross attention
					_snake_case              =      hidden_states
					_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					# feed forward
					_snake_case              =      hidden_states
					_snake_case              =      self.ff(self.norma(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      42
		A_						 =      42
		A_						 =      1
		A_						 =      0.0
		A_						 =      False
		A_						 =      False
		A_						 =      jnp.floataa
		A_						 =      False



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					_snake_case              =      nn.GroupNorm(num_groups=32  ,					epsilon=1e-5   )

					_snake_case              =      self.n_heads * self.d_head
					if self.use_linear_projection:
								_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype   )
					else:
								_snake_case              =      nn.Conv(
								    __SCREAMING_SNAKE_CASE  ,					kernel_size=(1, 1)  ,					strides=(1, 1)  ,					padding="VALID"  ,					dtype=self.dtype  ,					)

					_snake_case              =      [
					    FlaxBasicTransformerBlock(
					        __SCREAMING_SNAKE_CASE  ,					self.n_heads  ,					self.d_head  ,					dropout=self.dropout  ,					only_cross_attention=self.only_cross_attention  ,					dtype=self.dtype  ,					use_memory_efficient_attention=self.use_memory_efficient_attention  ,					)
					    for _ in range(self.depth   )
					]

					if self.use_linear_projection:
								_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype   )
					else:
								_snake_case              =      nn.Conv(
								    __SCREAMING_SNAKE_CASE  ,					kernel_size=(1, 1)  ,					strides=(1, 1)  ,					padding="VALID"  ,					dtype=self.dtype  ,					)

					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]=True   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case       , _snake_case              =      hidden_states.shape
					_snake_case              =      hidden_states
					_snake_case              =      self.norm(__SCREAMING_SNAKE_CASE   )
					if self.use_linear_projection:
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					height * width  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      self.proj_in(__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      self.proj_in(__SCREAMING_SNAKE_CASE   )
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					height * width  ,					__SCREAMING_SNAKE_CASE   )

					for transformer_block in self.transformer_blocks:
								_snake_case              =      transformer_block(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

					if self.use_linear_projection:
								_snake_case              =      self.proj_out(__SCREAMING_SNAKE_CASE   )
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      self.proj_out(__SCREAMING_SNAKE_CASE   )

					_snake_case              =      hidden_states + residual
					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      0.0
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Optional[int]   ):






					"""simple docstring"""
					# The second linear layer needs to be called
					# net_2 for now to match the index of the Sequential layer
					_snake_case              =      FlaxGEGLU(self.dim  ,					self.dropout  ,					self.dtype   )
					_snake_case              =      nn.Dense(self.dim  ,					dtype=self.dtype   )







		def __call__(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				List[Any]=True   ):






					"""simple docstring"""
					_snake_case              =      self.net_a(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.net_a(__SCREAMING_SNAKE_CASE   )
					return hidden_states

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      0.0
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Optional[Any]   ):






					"""simple docstring"""
					_snake_case              =      self.dim * 4
					_snake_case              =      nn.Dense(inner_dim * 2  ,					dtype=self.dtype   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]  ,					__SCREAMING_SNAKE_CASE    :				List[str]=True   ):






					"""simple docstring"""
					_snake_case              =      self.proj(__SCREAMING_SNAKE_CASE   )
					_snake_case       , _snake_case              =      jnp.split(__SCREAMING_SNAKE_CASE  ,					2  ,					axis=2   )
					return self.dropout_layer(hidden_linear * nn.gelu(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )

