

"""simple docstring"""






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp
def       UpperCamelCase					(			_lowerCAmelCase     :						str,  _lowerCAmelCase     :						Optional[Any],  _lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						int = 4096 )				->			List[str]:
 _UpperCAmelCase    :							Tuple									=					key.shape[-3:]
 _UpperCAmelCase    :							List[str]									=					value.shape[-1]
 _UpperCAmelCase    :							str									=					min(_lowerCAmelCase,  _lowerCAmelCase )
 _UpperCAmelCase    :							Tuple									=					query / jnp.sqrt(_lowerCAmelCase )

 @functools.partial(jax.checkpoint,  prevent_cse=_lowerCAmelCase )
 def summarize_chunk(_lowerCAmelCase     :						int,  _lowerCAmelCase     :						Dict,  _lowerCAmelCase     :						Union[str, Any] ):
  _UpperCAmelCase    :							str									=					jnp.einsum("""...qhd,...khd->...qhk""",  _lowerCAmelCase,  _lowerCAmelCase,  precision=_lowerCAmelCase )

  _UpperCAmelCase    :							Any									=					jnp.max(_lowerCAmelCase,  axis=-1,  keepdims=_lowerCAmelCase )
  _UpperCAmelCase    :							str									=					jax.lax.stop_gradient(_lowerCAmelCase )
  _UpperCAmelCase    :							Optional[int]									=					jnp.exp(attn_weights - max_score )

  _UpperCAmelCase    :							Optional[int]									=					jnp.einsum("""...vhf,...qhv->...qhf""",  _lowerCAmelCase,  _lowerCAmelCase,  precision=_lowerCAmelCase )
  _UpperCAmelCase    :							int									=					jnp.einsum("""...qhk->...qh""",  _lowerCAmelCase )

  return (exp_values, exp_weights.sum(axis=-1 ), max_score)

 def chunk_scanner(_lowerCAmelCase     :						List[Any] ):
  # julienne key array
  _UpperCAmelCase    :							Any									=					jax.lax.dynamic_slice(
      operand=_lowerCAmelCase,  start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0],  slice_sizes=list(key.shape[:-3] ) + [key_chunk_size, num_heads, k_features],  )

  # julienne value array
  _UpperCAmelCase    :							List[str]									=					jax.lax.dynamic_slice(
      operand=_lowerCAmelCase,  start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0],  slice_sizes=list(value.shape[:-3] ) + [key_chunk_size, num_heads, v_features],  )

  return summarize_chunk(_lowerCAmelCase,  _lowerCAmelCase,  _lowerCAmelCase )

 _UpperCAmelCase    :							Any									=					jax.lax.map(f=_lowerCAmelCase,  xs=jnp.arange(0,  _lowerCAmelCase,  _lowerCAmelCase ) )

 _UpperCAmelCase    :							List[Any]									=					jnp.max(_lowerCAmelCase,  axis=0,  keepdims=_lowerCAmelCase )
 _UpperCAmelCase    :							Tuple									=					jnp.exp(chunk_max - global_max )

 chunk_values *= jnp.expand_dims(_lowerCAmelCase,  axis=-1 )
 chunk_weights *= max_diffs

 _UpperCAmelCase    :							Any									=					chunk_values.sum(axis=0 )
 _UpperCAmelCase    :							int									=					jnp.expand_dims(_lowerCAmelCase,  -1 ).sum(axis=0 )

 return all_values / all_weights
def       UpperCamelCase					(			_lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						Union[str, Any],  _lowerCAmelCase     :						Optional[Any],  _lowerCAmelCase     :						Optional[Any]=jax.lax.Precision.HIGHEST,  _lowerCAmelCase     :						int = 1024,  _lowerCAmelCase     :						int = 4096 )				->			Union[str, Any]:
 _UpperCAmelCase    :							Union[str, Any]									=					query.shape[-3:]

 def chunk_scanner(_lowerCAmelCase     :						str,  _lowerCAmelCase     :						str ):
  # julienne query array
  _UpperCAmelCase    :							List[str]									=					jax.lax.dynamic_slice(
      operand=_lowerCAmelCase,  start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0],  slice_sizes=list(query.shape[:-3] ) + [min(_lowerCAmelCase,  _lowerCAmelCase ), num_heads, q_features],  )

  return (
      chunk_idx + query_chunk_size,  # unused ignore it
      _query_chunk_attention(
          query=_lowerCAmelCase,  key=_lowerCAmelCase,  value=_lowerCAmelCase,  precision=_lowerCAmelCase,  key_chunk_size=_lowerCAmelCase ),
  )

 _UpperCAmelCase    :							List[Any]									=					jax.lax.scan(
     f=_lowerCAmelCase,  init=0,  xs=_lowerCAmelCase,  length=math.ceil(num_q / query_chunk_size ) )

 return jnp.concatenate(_lowerCAmelCase,  axis=-3 )  # fuse the chunked result back






class     _UpperCAmelCase			(		nn.Module):
   __a :       int
   __a :       int									=							8
   __a :       int									=							6_4
   __a :       float									=							0.0
   __a :       bool									=							False
   __a :       jnp.dtype									=							jnp.floataa
   def 							__snake_case (		self     )    ->    Optional[int]:






    '''simple docstring'''







    _UpperCAmelCase    :							str									=					self.dim_head * self.heads
    _UpperCAmelCase    :							List[Any]									=					self.dim_head**-0.5

    # Weights were exported with old names {to_q, to_k, to_v, to_out}
    _UpperCAmelCase    :							Union[str, Any]									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_q"""     )
    _UpperCAmelCase    :							str									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_k"""     )
    _UpperCAmelCase    :							Optional[Any]									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_v"""     )

    _UpperCAmelCase    :							List[Any]									=					nn.Dense(self.query_dim      ,		dtype=self.dtype      ,		name="""to_out_0"""     )
    _UpperCAmelCase    :							Any									=					nn.Dropout(rate=self.dropout     )
   def 							__snake_case (		self      ,		_A     )    ->    int:






    '''simple docstring'''







    _UpperCAmelCase    :							Any									=					tensor.shape
    _UpperCAmelCase    :							Union[str, Any]									=					self.heads
    _UpperCAmelCase    :							List[str]									=					tensor.reshape(_A      ,		_A      ,		_A      ,		dim // head_size     )
    _UpperCAmelCase    :							Optional[int]									=					jnp.transpose(_A      ,		(0, 2, 1, 3)     )
    _UpperCAmelCase    :							Optional[Any]									=					tensor.reshape(batch_size * head_size      ,		_A      ,		dim // head_size     )
    return tensor
   def 							__snake_case (		self      ,		_A     )    ->    Union[str, Any]:






    '''simple docstring'''







    _UpperCAmelCase    :							List[Any]									=					tensor.shape
    _UpperCAmelCase    :							Optional[int]									=					self.heads
    _UpperCAmelCase    :							str									=					tensor.reshape(batch_size // head_size      ,		_A      ,		_A      ,		_A     )
    _UpperCAmelCase    :							Optional[Any]									=					jnp.transpose(_A      ,		(0, 2, 1, 3)     )
    _UpperCAmelCase    :							Tuple									=					tensor.reshape(batch_size // head_size      ,		_A      ,		dim * head_size     )
    return tensor
   def __call__(		self      ,		_A      ,		_A=None      ,		_A=True     )    ->    Dict:






    '''simple docstring'''







    _UpperCAmelCase    :							Optional[int]									=					hidden_states if context is None else context

    _UpperCAmelCase    :							List[Any]									=					self.query(_A     )
    _UpperCAmelCase    :							List[Any]									=					self.key(_A     )
    _UpperCAmelCase    :							Union[str, Any]									=					self.value(_A     )

    _UpperCAmelCase    :							List[str]									=					self.reshape_heads_to_batch_dim(_A     )
    _UpperCAmelCase    :							List[Any]									=					self.reshape_heads_to_batch_dim(_A     )
    _UpperCAmelCase    :							List[Any]									=					self.reshape_heads_to_batch_dim(_A     )

    if self.use_memory_efficient_attention:
     _UpperCAmelCase    :							List[Any]									=					query_states.transpose(1      ,		0      ,		2     )
     _UpperCAmelCase    :							Any									=					key_states.transpose(1      ,		0      ,		2     )
     _UpperCAmelCase    :							Optional[int]									=					value_states.transpose(1      ,		0      ,		2     )

     # this if statement create a chunk size for each layer of the unet
     # the chunk size is equal to the query_length dimension of the deepest layer of the unet

     _UpperCAmelCase    :							int									=					query_states.shape[-3]
     if flatten_latent_dim % 64 == 0:
      _UpperCAmelCase    :							Union[str, Any]									=					int(flatten_latent_dim / 64     )
     elif flatten_latent_dim % 16 == 0:
      _UpperCAmelCase    :							List[str]									=					int(flatten_latent_dim / 16     )
     elif flatten_latent_dim % 4 == 0:
      _UpperCAmelCase    :							Optional[int]									=					int(flatten_latent_dim / 4     )
     else:
      _UpperCAmelCase    :							Union[str, Any]									=					int(_A     )

     _UpperCAmelCase    :							Dict									=					jax_memory_efficient_attention(
         _A      ,		_A      ,		_A      ,		query_chunk_size=_A      ,		key_chunk_size=40_96 * 4     )

     _UpperCAmelCase    :							Dict									=					hidden_states.transpose(1      ,		0      ,		2     )
    else:
     # compute attentions
     _UpperCAmelCase    :							List[Any]									=					jnp.einsum("""b i d, b j d->b i j"""      ,		_A      ,		_A     )
     _UpperCAmelCase    :							List[str]									=					attention_scores * self.scale
     _UpperCAmelCase    :							Union[str, Any]									=					nn.softmax(_A      ,		axis=2     )

     # attend to values
     _UpperCAmelCase    :							Tuple									=					jnp.einsum("""b i j, b j d -> b i d"""      ,		_A      ,		_A     )

    _UpperCAmelCase    :							int									=					self.reshape_batch_dim_to_heads(_A     )
    _UpperCAmelCase    :							str									=					self.proj_attn(_A     )
    return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
   __a :       int
   __a :       int
   __a :       int
   __a :       float									=							0.0
   __a :       bool									=							False
   __a :       jnp.dtype									=							jnp.floataa
   __a :       bool									=							False
   def 							__snake_case (		self     )    ->    Optional[Any]:






    '''simple docstring'''







    _UpperCAmelCase    :							Any									=					FlaxAttention(
        self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype     )
    # cross attention
    _UpperCAmelCase    :							Optional[int]									=					FlaxAttention(
        self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							Optional[int]									=					FlaxFeedForward(dim=self.dim      ,		dropout=self.dropout      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							str									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							Any									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							Any									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							Dict									=					nn.Dropout(rate=self.dropout     )
   def __call__(		self      ,		_A      ,		_A      ,		_A=True     )    ->    Any:






    '''simple docstring'''







    _UpperCAmelCase    :							Any									=					hidden_states
    if self.only_cross_attention:
     _UpperCAmelCase    :							Tuple									=					self.attna(self.norma(_A     )      ,		_A      ,		deterministic=_A     )
    else:
     _UpperCAmelCase    :							str									=					self.attna(self.norma(_A     )      ,		deterministic=_A     )
    _UpperCAmelCase    :							Dict									=					hidden_states + residual

    # cross attention
    _UpperCAmelCase    :							List[Any]									=					hidden_states
    _UpperCAmelCase    :							List[Any]									=					self.attna(self.norma(_A     )      ,		_A      ,		deterministic=_A     )
    _UpperCAmelCase    :							int									=					hidden_states + residual

    # feed forward
    _UpperCAmelCase    :							str									=					hidden_states
    _UpperCAmelCase    :							Union[str, Any]									=					self.ff(self.norma(_A     )      ,		deterministic=_A     )
    _UpperCAmelCase    :							Union[str, Any]									=					hidden_states + residual

    return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
   __a :       int
   __a :       int
   __a :       int
   __a :       int									=							1
   __a :       float									=							0.0
   __a :       bool									=							False
   __a :       bool									=							False
   __a :       jnp.dtype									=							jnp.floataa
   __a :       bool									=							False
   def 							__snake_case (		self     )    ->    str:






    '''simple docstring'''







    _UpperCAmelCase    :							Any									=					nn.GroupNorm(num_groups=32      ,		epsilon=1e-5     )

    _UpperCAmelCase    :							Optional[Any]									=					self.n_heads * self.d_head
    if self.use_linear_projection:
     _UpperCAmelCase    :							Optional[Any]									=					nn.Dense(_A      ,		dtype=self.dtype     )
    else:
     _UpperCAmelCase    :							List[Any]									=					nn.Conv(
         _A      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding="""VALID"""      ,		dtype=self.dtype      ,		)

    _UpperCAmelCase    :							Optional[int]									=					[
        FlaxBasicTransformerBlock(
            _A      ,		self.n_heads      ,		self.d_head      ,		dropout=self.dropout      ,		only_cross_attention=self.only_cross_attention      ,		dtype=self.dtype      ,		use_memory_efficient_attention=self.use_memory_efficient_attention      ,		)
        for _ in range(self.depth     )
    ]

    if self.use_linear_projection:
     _UpperCAmelCase    :							Union[str, Any]									=					nn.Dense(_A      ,		dtype=self.dtype     )
    else:
     _UpperCAmelCase    :							Optional[Any]									=					nn.Conv(
         _A      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding="""VALID"""      ,		dtype=self.dtype      ,		)

    _UpperCAmelCase    :							List[Any]									=					nn.Dropout(rate=self.dropout     )
   def __call__(		self      ,		_A      ,		_A      ,		_A=True     )    ->    Any:






    '''simple docstring'''







    _UpperCAmelCase    :							int									=					hidden_states.shape
    _UpperCAmelCase    :							Tuple									=					hidden_states
    _UpperCAmelCase    :							Optional[Any]									=					self.norm(_A     )
    if self.use_linear_projection:
     _UpperCAmelCase    :							Dict									=					hidden_states.reshape(_A      ,		height * width      ,		_A     )
     _UpperCAmelCase    :							List[Any]									=					self.proj_in(_A     )
    else:
     _UpperCAmelCase    :							Union[str, Any]									=					self.proj_in(_A     )
     _UpperCAmelCase    :							List[Any]									=					hidden_states.reshape(_A      ,		height * width      ,		_A     )

    for transformer_block in self.transformer_blocks:
     _UpperCAmelCase    :							int									=					transformer_block(_A      ,		_A      ,		deterministic=_A     )

    if self.use_linear_projection:
     _UpperCAmelCase    :							Optional[int]									=					self.proj_out(_A     )
     _UpperCAmelCase    :							Dict									=					hidden_states.reshape(_A      ,		_A      ,		_A      ,		_A     )
    else:
     _UpperCAmelCase    :							str									=					hidden_states.reshape(_A      ,		_A      ,		_A      ,		_A     )
     _UpperCAmelCase    :							Union[str, Any]									=					self.proj_out(_A     )

    _UpperCAmelCase    :							List[Any]									=					hidden_states + residual
    return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
   __a :       int
   __a :       float									=							0.0
   __a :       jnp.dtype									=							jnp.floataa
   def 							__snake_case (		self     )    ->    Any:






    '''simple docstring'''







    _UpperCAmelCase    :							Optional[int]									=					FlaxGEGLU(self.dim      ,		self.dropout      ,		self.dtype     )
    _UpperCAmelCase    :							Dict									=					nn.Dense(self.dim      ,		dtype=self.dtype     )
   def __call__(		self      ,		_A      ,		_A=True     )    ->    Union[str, Any]:






    '''simple docstring'''







    _UpperCAmelCase    :							Dict									=					self.net_a(_A      ,		deterministic=_A     )
    _UpperCAmelCase    :							Tuple									=					self.net_a(_A     )
    return hidden_states






class     _UpperCAmelCase			(		nn.Module):
   __a :       int
   __a :       float									=							0.0
   __a :       jnp.dtype									=							jnp.floataa
   def 							__snake_case (		self     )    ->    Tuple:






    '''simple docstring'''







    _UpperCAmelCase    :							Dict									=					self.dim * 4
    _UpperCAmelCase    :							Optional[Any]									=					nn.Dense(inner_dim * 2      ,		dtype=self.dtype     )
    _UpperCAmelCase    :							int									=					nn.Dropout(rate=self.dropout     )
   def __call__(		self      ,		_A      ,		_A=True     )    ->    int:






    '''simple docstring'''







    _UpperCAmelCase    :							Optional[Any]									=					self.proj(_A     )
    _UpperCAmelCase    :							Optional[int]									=					jnp.split(_A      ,		2      ,		axis=2     )
    return self.dropout_layer(hidden_linear * nn.gelu(_A     )      ,		deterministic=_A     )








"""simple docstring"""






import safs






class     _UpperCAmelCase			(		safs.SaFileSystem):
			pass





