


import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp
def        __lowerCamelCase		(				UpperCAmelCase_		:	Union[str, Any]				,						UpperCAmelCase_		:	List[str]				,						UpperCAmelCase_		:	int				,						UpperCAmelCase_		:	List[Any]				,						UpperCAmelCase_		:	str = 4096		):




						"""simple docstring"""





						a		:Optional[Any]											= key.shape[-3:]
						a		:List[Any]											= value.shape[-1]
						a		:str											= min(__lowerCamelCase				,						__lowerCamelCase		)
						a		:str											= query / jnp.sqrt(__lowerCamelCase		)

						@functools.partial(jax.checkpoint				,						prevent_cse=__lowerCamelCase		)
						def summarize_chunk(UpperCAmelCase_		:	List[Any]				,						UpperCAmelCase_		:	Optional[Any]				,						UpperCAmelCase_		:	int		):
												a		:Optional[int]											= jnp.einsum('''...qhd,...khd->...qhk'''				,						__lowerCamelCase				,						__lowerCamelCase				,						precision=__lowerCamelCase		)

												a		:Optional[int]											= jnp.max(__lowerCamelCase				,						axis=-1				,						keepdims=__lowerCamelCase		)
												a		:Union[str, Any]											= jax.lax.stop_gradient(__lowerCamelCase		)
												a		:str											= jnp.exp(attn_weights - max_score		)

												a		:Any											= jnp.einsum('''...vhf,...qhv->...qhf'''				,						__lowerCamelCase				,						__lowerCamelCase				,						precision=__lowerCamelCase		)
												a		:Optional[int]											= jnp.einsum('''...qhk->...qh'''				,						__lowerCamelCase		)

												return (exp_values, exp_weights.sum(axis=-1		), max_score)

						def chunk_scanner(UpperCAmelCase_		:	Any		):
												# julienne key array
												a		:Union[str, Any]											= jax.lax.dynamic_slice(
												    operand=__lowerCamelCase				,						start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]				,						slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]				,						)

												# julienne value array
												a		:Dict											= jax.lax.dynamic_slice(
												    operand=__lowerCamelCase				,						start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]				,						slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]				,						)

												return summarize_chunk(__lowerCamelCase				,						__lowerCamelCase				,						__lowerCamelCase		)

						a		:int											= jax.lax.map(f=__lowerCamelCase				,						xs=jnp.arange(0				,						__lowerCamelCase				,						__lowerCamelCase		)		)

						a		:Optional[Any]											= jnp.max(__lowerCamelCase				,						axis=0				,						keepdims=__lowerCamelCase		)
						a		:Dict											= jnp.exp(chunk_max - global_max		)

						chunk_values *= jnp.expand_dims(__lowerCamelCase				,						axis=-1		)
						chunk_weights *= max_diffs

						a		:Tuple											= chunk_values.sum(axis=0		)
						a		:List[str]											= jnp.expand_dims(__lowerCamelCase				,						-1		).sum(axis=0		)

						return all_values / all_weights
def        __lowerCamelCase		(				UpperCAmelCase_		:	Union[str, Any]				,						UpperCAmelCase_		:	List[Any]				,						UpperCAmelCase_		:	int				,						UpperCAmelCase_		:	List[str]=jax.lax.Precision.HIGHEST				,						UpperCAmelCase_		:	List[str] = 1024				,						UpperCAmelCase_		:	Union[str, Any] = 4096		):




						"""simple docstring"""





						a		:str											= query.shape[-3:]

						def chunk_scanner(UpperCAmelCase_		:	Union[str, Any]				,						UpperCAmelCase_		:	Optional[int]		):
												# julienne query array
												a		:Optional[int]											= jax.lax.dynamic_slice(
												    operand=__lowerCamelCase				,						start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]				,						slice_sizes=list(query.shape[:-3]		) + [min(__lowerCamelCase				,						__lowerCamelCase		), num_heads, q_features]				,						)

												return (
												    chunk_idx + query_chunk_size,  # unused ignore it
												    _query_chunk_attention(
												        query=__lowerCamelCase				,						key=__lowerCamelCase				,						value=__lowerCamelCase				,						precision=__lowerCamelCase				,						key_chunk_size=__lowerCamelCase		),
												)

						a		:int											= jax.lax.scan(
						    f=__lowerCamelCase				,						init=0				,						xs=__lowerCamelCase				,						length=math.ceil(num_q / query_chunk_size		)		)

						return jnp.concatenate(__lowerCamelCase				,						axis=-3		)  # fuse the chunked result back







class 	_snake_case						(   nn.Module							):
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 8
	SCREAMING_SNAKE_CASE__   = 64
	SCREAMING_SNAKE_CASE__   = 0.0
	SCREAMING_SNAKE_CASE__   = False
	SCREAMING_SNAKE_CASE__   = jnp.floataa




	def  SCREAMING_SNAKE_CASE__       (	self					):
							a		:str											= self.dim_head * self.heads
							a		:Optional[Any]											= self.dim_head**-0.5

							# Weights were exported with old names {to_q, to_k, to_v, to_out}
							a		:Any											= nn.Dense(_lowerCAmelCase      ,						use_bias=_lowerCAmelCase      ,						dtype=self.dtype      ,						name='''to_q'''					)
							a		:int											= nn.Dense(_lowerCAmelCase      ,						use_bias=_lowerCAmelCase      ,						dtype=self.dtype      ,						name='''to_k'''					)
							a		:Union[str, Any]											= nn.Dense(_lowerCAmelCase      ,						use_bias=_lowerCAmelCase      ,						dtype=self.dtype      ,						name='''to_v'''					)

							a		:int											= nn.Dense(self.query_dim      ,						dtype=self.dtype      ,						name='''to_out_0'''					)
							a		:Union[str, Any]											= nn.Dropout(rate=self.dropout					)




	def  SCREAMING_SNAKE_CASE__       (	self      ,						_lowerCamelCase					):
							a		:Optional[Any]											= tensor.shape
							a		:Tuple											= self.heads
							a		:Tuple											= tensor.reshape(_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase      ,						dim // head_size					)
							a		:Tuple											= jnp.transpose(_lowerCAmelCase      ,						(0, 2, 1, 3)					)
							a		:str											= tensor.reshape(batch_size * head_size      ,						_lowerCAmelCase      ,						dim // head_size					)
							return tensor




	def  SCREAMING_SNAKE_CASE__       (	self      ,						_lowerCamelCase					):
							a		:Union[str, Any]											= tensor.shape
							a		:List[Any]											= self.heads
							a		:Optional[Any]											= tensor.reshape(batch_size // head_size      ,						_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase					)
							a		:List[Any]											= jnp.transpose(_lowerCAmelCase      ,						(0, 2, 1, 3)					)
							a		:List[Any]											= tensor.reshape(batch_size // head_size      ,						_lowerCAmelCase      ,						dim * head_size					)
							return tensor




	def __call__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True					):
							a		:Union[str, Any]											= hidden_states if context is None else context

							a		:int											= self.query(_lowerCAmelCase					)
							a		:List[str]											= self.key(_lowerCAmelCase					)
							a		:Union[str, Any]											= self.value(_lowerCAmelCase					)

							a		:Union[str, Any]											= self.reshape_heads_to_batch_dim(_lowerCAmelCase					)
							a		:Union[str, Any]											= self.reshape_heads_to_batch_dim(_lowerCAmelCase					)
							a		:Dict											= self.reshape_heads_to_batch_dim(_lowerCAmelCase					)

							if self.use_memory_efficient_attention:
													a		:List[str]											= query_states.transpose(1      ,						0      ,						2					)
													a		:Tuple											= key_states.transpose(1      ,						0      ,						2					)
													a		:List[Any]											= value_states.transpose(1      ,						0      ,						2					)

													# this if statement create a chunk size for each layer of the unet
													# the chunk size is equal to the query_length dimension of the deepest layer of the unet

													a		:List[str]											= query_states.shape[-3]
													if flatten_latent_dim % 64 == 0:
																			a		:Dict											= int(flatten_latent_dim / 64					)
													elif flatten_latent_dim % 16 == 0:
																			a		:int											= int(flatten_latent_dim / 16					)
													elif flatten_latent_dim % 4 == 0:
																			a		:str											= int(flatten_latent_dim / 4					)
													else:
																			a		:Tuple											= int(_lowerCAmelCase					)

													a		:int											= jax_memory_efficient_attention(
													    _lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase      ,						query_chunk_size=_lowerCAmelCase      ,						key_chunk_size=4096 * 4					)

													a		:List[Any]											= hidden_states.transpose(1      ,						0      ,						2					)
							else:
													# compute attentions
													a		:Tuple											= jnp.einsum('''b i d, b j d->b i j'''      ,						_lowerCAmelCase      ,						_lowerCAmelCase					)
													a		:Any											= attention_scores * self.scale
													a		:Optional[int]											= nn.softmax(_lowerCAmelCase      ,						axis=2					)

													# attend to values
													a		:Any											= jnp.einsum('''b i j, b j d -> b i d'''      ,						_lowerCAmelCase      ,						_lowerCAmelCase					)

							a		:Union[str, Any]											= self.reshape_batch_dim_to_heads(_lowerCAmelCase					)
							a		:List[str]											= self.proj_attn(_lowerCAmelCase					)
							return self.dropout_layer(_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)







class 	_snake_case						(   nn.Module							):
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 0.0
	SCREAMING_SNAKE_CASE__   = False
	SCREAMING_SNAKE_CASE__   = jnp.floataa
	SCREAMING_SNAKE_CASE__   = False




	def  SCREAMING_SNAKE_CASE__       (	self					):
							# self attention (or cross_attention if only_cross_attention is True)
							a		:List[Any]											= FlaxAttention(
							    self.dim      ,						self.n_heads      ,						self.d_head      ,						self.dropout      ,						self.use_memory_efficient_attention      ,						dtype=self.dtype					)
							# cross attention
							a		:List[str]											= FlaxAttention(
							    self.dim      ,						self.n_heads      ,						self.d_head      ,						self.dropout      ,						self.use_memory_efficient_attention      ,						dtype=self.dtype					)
							a		:Any											= FlaxFeedForward(dim=self.dim      ,						dropout=self.dropout      ,						dtype=self.dtype					)
							a		:Tuple											= nn.LayerNorm(epsilon=1e-5      ,						dtype=self.dtype					)
							a		:Optional[Any]											= nn.LayerNorm(epsilon=1e-5      ,						dtype=self.dtype					)
							a		:Any											= nn.LayerNorm(epsilon=1e-5      ,						dtype=self.dtype					)
							a		:Union[str, Any]											= nn.Dropout(rate=self.dropout					)




	def __call__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase      ,						_lowerCamelCase=True					):
							# self attention
							a		:List[str]											= hidden_states
							if self.only_cross_attention:
													a		:Dict											= self.attna(self.norma(_lowerCAmelCase					)      ,						_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)
							else:
													a		:Any											= self.attna(self.norma(_lowerCAmelCase					)      ,						deterministic=_lowerCAmelCase					)
							a		:List[Any]											= hidden_states + residual

							# cross attention
							a		:Optional[int]											= hidden_states
							a		:int											= self.attna(self.norma(_lowerCAmelCase					)      ,						_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)
							a		:Any											= hidden_states + residual

							# feed forward
							a		:Union[str, Any]											= hidden_states
							a		:str											= self.ff(self.norma(_lowerCAmelCase					)      ,						deterministic=_lowerCAmelCase					)
							a		:int											= hidden_states + residual

							return self.dropout_layer(_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)







class 	_snake_case						(   nn.Module							):
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 1
	SCREAMING_SNAKE_CASE__   = 0.0
	SCREAMING_SNAKE_CASE__   = False
	SCREAMING_SNAKE_CASE__   = False
	SCREAMING_SNAKE_CASE__   = jnp.floataa
	SCREAMING_SNAKE_CASE__   = False




	def  SCREAMING_SNAKE_CASE__       (	self					):
							a		:Dict											= nn.GroupNorm(num_groups=32      ,						epsilon=1e-5					)

							a		:Dict											= self.n_heads * self.d_head
							if self.use_linear_projection:
													a		:Tuple											= nn.Dense(_lowerCAmelCase      ,						dtype=self.dtype					)
							else:
													a		:Dict											= nn.Conv(
													    _lowerCAmelCase      ,						kernel_size=(1, 1)      ,						strides=(1, 1)      ,						padding='''VALID'''      ,						dtype=self.dtype      ,						)

							a		:int											= [
							    FlaxBasicTransformerBlock(
							        _lowerCAmelCase      ,						self.n_heads      ,						self.d_head      ,						dropout=self.dropout      ,						only_cross_attention=self.only_cross_attention      ,						dtype=self.dtype      ,						use_memory_efficient_attention=self.use_memory_efficient_attention      ,						)
							    for _ in range(self.depth					)
							]

							if self.use_linear_projection:
													a		:Union[str, Any]											= nn.Dense(_lowerCAmelCase      ,						dtype=self.dtype					)
							else:
													a		:Union[str, Any]											= nn.Conv(
													    _lowerCAmelCase      ,						kernel_size=(1, 1)      ,						strides=(1, 1)      ,						padding='''VALID'''      ,						dtype=self.dtype      ,						)

							a		:Union[str, Any]											= nn.Dropout(rate=self.dropout					)




	def __call__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase      ,						_lowerCamelCase=True					):
							a		:Optional[int]											= hidden_states.shape
							a		:Union[str, Any]											= hidden_states
							a		:Union[str, Any]											= self.norm(_lowerCAmelCase					)
							if self.use_linear_projection:
													a		:int											= hidden_states.reshape(_lowerCAmelCase      ,						height * width      ,						_lowerCAmelCase					)
													a		:int											= self.proj_in(_lowerCAmelCase					)
							else:
													a		:Any											= self.proj_in(_lowerCAmelCase					)
													a		:List[str]											= hidden_states.reshape(_lowerCAmelCase      ,						height * width      ,						_lowerCAmelCase					)

							for transformer_block in self.transformer_blocks:
													a		:str											= transformer_block(_lowerCAmelCase      ,						_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)

							if self.use_linear_projection:
													a		:Optional[int]											= self.proj_out(_lowerCAmelCase					)
													a		:Tuple											= hidden_states.reshape(_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase					)
							else:
													a		:str											= hidden_states.reshape(_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase      ,						_lowerCAmelCase					)
													a		:Union[str, Any]											= self.proj_out(_lowerCAmelCase					)

							a		:List[str]											= hidden_states + residual
							return self.dropout_layer(_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)







class 	_snake_case						(   nn.Module							):
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 0.0
	SCREAMING_SNAKE_CASE__   = jnp.floataa




	def  SCREAMING_SNAKE_CASE__       (	self					):
							# The second linear layer needs to be called
							# net_2 for now to match the index of the Sequential layer
							a		:Dict											= FlaxGEGLU(self.dim      ,						self.dropout      ,						self.dtype					)
							a		:Optional[int]											= nn.Dense(self.dim      ,						dtype=self.dtype					)




	def __call__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase=True					):
							a		:Optional[int]											= self.net_a(_lowerCAmelCase      ,						deterministic=_lowerCAmelCase					)
							a		:List[str]											= self.net_a(_lowerCAmelCase					)
							return hidden_states







class 	_snake_case						(   nn.Module							):
	SCREAMING_SNAKE_CASE__   = 42
	SCREAMING_SNAKE_CASE__   = 0.0
	SCREAMING_SNAKE_CASE__   = jnp.floataa




	def  SCREAMING_SNAKE_CASE__       (	self					):
							a		:List[Any]											= self.dim * 4
							a		:Any											= nn.Dense(inner_dim * 2      ,						dtype=self.dtype					)
							a		:Dict											= nn.Dropout(rate=self.dropout					)




	def __call__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase=True					):
							a		:List[Any]											= self.proj(_lowerCAmelCase					)
							a		:int											= jnp.split(_lowerCAmelCase      ,						2      ,						axis=2					)
							return self.dropout_layer(hidden_linear * nn.gelu(_lowerCAmelCase					)      ,						deterministic=_lowerCAmelCase					)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


snake_case       :					Dict        =  {
    '''configuration_align''': [
        '''ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AlignConfig''',
        '''AlignTextConfig''',
        '''AlignVisionConfig''',
    ],
    '''processing_align''': ['''AlignProcessor'''],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     snake_case       :					List[str]        =  [
         '''ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''AlignModel''',
         '''AlignPreTrainedModel''',
         '''AlignTextModel''',
         '''AlignVisionModel''',
     ]

if TYPE_CHECKING:
     from .configuration_align import (
         ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
         AlignConfig,
         AlignTextConfig,
         AlignVisionConfig,
     )
     from .processing_align import AlignProcessor

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_align import (
              ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
              AlignModel,
              AlignPreTrainedModel,
              AlignTextModel,
              AlignVisionModel,
          )

else:
     import sys

     snake_case       :					int        =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





