
"""simple docstring"""






import os

from apache_beam.io.filesystems import FileSystems
from apache_beam.pipeline import Pipeline

from .logging import get_logger


_lowercase							 =							2 << 20  # 2mb
_lowercase							 =							get_logger(__name__)

class 				__a    (   lowerCAmelCase_					):



       '''simple docstring'''







       def      SCREAMING_SNAKE_CASE	(    self					) ->       Optional[int]:




          '''simple docstring'''
          __lowercase						      =					self._options.get_all_options().get("runner"					)
          return runner in [None, "DirectRunner", "PortableRunner"]





def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__=False			)	->List[Any]:
   __lowercase						      =					FileSystems
   if fs.exists(a_			):
      if force_upload:
         logger.info(F'''Remote path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''			)
      else:
         logger.info(F'''Remote path already exist: {remote_file_path}. Skipping it as force_upload=False.'''			)
         return
   with fs.create(a_			) as remote_file:
      with open(a_ ,      "rb"			) as local_file:
         __lowercase						      =					local_file.read(a_			)
         while chunk:
            remote_file.write(a_			)
            __lowercase						      =					local_file.read(a_			)







def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__=False			)	->Dict:
   __lowercase						      =					FileSystems
   if os.path.exists(a_			):
      if force_download:
         logger.info(F'''Local path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''			)
      else:
         logger.info(F'''Local path already exist: {remote_file_path}. Skipping it as force_upload=False.'''			)
         return
   with fs.open(a_			) as remote_file:
      with open(a_ ,      "wb"			) as local_file:
         __lowercase						      =					remote_file.read(a_			)
         while chunk:
            local_file.write(a_			)
            __lowercase						      =					remote_file.read(a_			)







"""simple docstring"""






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ = 4_0_9_6			)	->Dict:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					key.shape[-3:]
			__lowercase						      =					value.shape[-1]
			__lowercase						      =					min(__magic_name__ ,      __magic_name__			)
			__lowercase						      =					query / jnp.sqrt(__magic_name__			)

			@functools.partial(jax.checkpoint ,      prevent_cse=__magic_name__			)
			def summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			):
						__lowercase						      =					jnp.einsum("...qhd,...khd->...qhk" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)

						__lowercase						      =					jnp.max(__magic_name__ ,      axis=-1 ,      keepdims=__magic_name__			)
						__lowercase						      =					jax.lax.stop_gradient(__magic_name__			)
						__lowercase						      =					jnp.exp(attn_weights - max_score			)

						__lowercase						      =					jnp.einsum("...vhf,...qhv->...qhf" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)
						__lowercase						      =					jnp.einsum("...qhk->...qh" ,      __magic_name__			)

						return (exp_values, exp_weights.sum(axis=-1			), max_score)

			def chunk_scanner(__magic_name__			):
						# julienne key array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(key.shape[:-3]			) + [key_chunk_size, num_heads, k_features] ,      )

						# julienne value array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(value.shape[:-3]			) + [key_chunk_size, num_heads, v_features] ,      )

						return summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			)

			__lowercase      ,				__lowercase      ,				__lowercase						      =					jax.lax.map(f=__magic_name__ ,      xs=jnp.arange(0 ,      __magic_name__ ,      __magic_name__			)			)

			__lowercase						      =					jnp.max(__magic_name__ ,      axis=0 ,      keepdims=__magic_name__			)
			__lowercase						      =					jnp.exp(chunk_max - global_max			)

			chunk_values *= jnp.expand_dims(__magic_name__ ,      axis=-1			)
			chunk_weights *= max_diffs

			__lowercase						      =					chunk_values.sum(axis=0			)
			__lowercase						      =					jnp.expand_dims(__magic_name__ ,      -1			).sum(axis=0			)

			return all_values / all_weights



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__=jax.lax.Precision.HIGHEST ,      __magic_name__ = 1_0_2_4 ,      __magic_name__ = 4_0_9_6			)	->List[str]:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					query.shape[-3:]

			def chunk_scanner(__magic_name__ ,      __magic_name__			):
						# julienne query array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0] ,      slice_sizes=list(query.shape[:-3]			) + [min(__magic_name__ ,      __magic_name__			), num_heads, q_features] ,      )

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=__magic_name__ ,      key=__magic_name__ ,      value=__magic_name__ ,      precision=__magic_name__ ,      key_chunk_size=__magic_name__			),
						)

			__lowercase      ,				__lowercase						      =					jax.lax.scan(
			    f=__magic_name__ ,      init=0 ,      xs=__magic_name__ ,      length=math.ceil(num_q / query_chunk_size			)			)

			return jnp.concatenate(__magic_name__ ,      axis=-3			)  # fuse the chunked result back

class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      8
							_lowerCamelCase				: int					      =      64
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       Any:




										'''simple docstring'''
										__lowercase						      =					self.dim_head * self.heads
										__lowercase						      =					self.dim_head**-0.5

										# Weights were exported with old names {to_q, to_k, to_v, to_out}
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_q"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_k"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_v"					)

										__lowercase						      =					nn.Dense(self.query_dim     ,						dtype=self.dtype     ,						name="to_out_0"					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       Optional[int]:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						dim // head_size					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size * head_size     ,						_lowerCamelCase     ,						dim // head_size					)
										return tensor
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       str:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						dim * head_size					)
										return tensor





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=None     ,						_lowerCamelCase=True					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					hidden_states if context is None else context

										__lowercase						      =					self.query(_lowerCamelCase					)
										__lowercase						      =					self.key(_lowerCamelCase					)
										__lowercase						      =					self.value(_lowerCamelCase					)

										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)

										if self.use_memory_efficient_attention:
													__lowercase						      =					query_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					key_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					value_states.transpose(1     ,						0     ,						2					)

													# this if statement create a chunk size for each layer of the unet
													# the chunk size is equal to the query_length dimension of the deepest layer of the unet

													__lowercase						      =					query_states.shape[-3]
													if flatten_latent_dim % 64 == 0:
																__lowercase						      =					int(flatten_latent_dim / 64					)
													elif flatten_latent_dim % 16 == 0:
																__lowercase						      =					int(flatten_latent_dim / 16					)
													elif flatten_latent_dim % 4 == 0:
																__lowercase						      =					int(flatten_latent_dim / 4					)
													else:
																__lowercase						      =					int(_lowerCamelCase					)

													__lowercase						      =					jax_memory_efficient_attention(
													    _lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						query_chunk_size=_lowerCamelCase     ,						key_chunk_size=4_096 * 4					)

													__lowercase						      =					hidden_states.transpose(1     ,						0     ,						2					)
										else:
													# compute attentions
													__lowercase						      =					jnp.einsum("b i d, b j d->b i j"     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					attention_scores * self.scale
													__lowercase						      =					nn.softmax(_lowerCamelCase     ,						axis=2					)

													# attend to values
													__lowercase						      =					jnp.einsum("b i j, b j d -> b i d"     ,						_lowerCamelCase     ,						_lowerCamelCase					)

										__lowercase						      =					self.reshape_batch_dim_to_heads(_lowerCamelCase					)
										__lowercase						      =					self.proj_attn(_lowerCamelCase					)
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# self attention (or cross_attention if only_cross_attention is True)
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										# cross attention
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										__lowercase						      =					FlaxFeedForward(dim=self.dim     ,						dropout=self.dropout     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Optional[int]:




										'''simple docstring'''
										# self attention
										__lowercase						      =					hidden_states
										if self.only_cross_attention:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										else:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# cross attention
										__lowercase						      =					hidden_states
										__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# feed forward
										__lowercase						      =					hidden_states
										__lowercase						      =					self.ff(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      1
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       int:




										'''simple docstring'''
										__lowercase						      =					nn.GroupNorm(num_groups=32     ,						epsilon=1e-5					)

										__lowercase						      =					self.n_heads * self.d_head
										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					[
										    FlaxBasicTransformerBlock(
										        _lowerCamelCase     ,						self.n_heads     ,						self.d_head     ,						dropout=self.dropout     ,						only_cross_attention=self.only_cross_attention     ,						dtype=self.dtype     ,						use_memory_efficient_attention=self.use_memory_efficient_attention     ,						)
										    for _ in range(self.depth					)
										]

										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase      ,				__lowercase						      =					hidden_states.shape
										__lowercase						      =					hidden_states
										__lowercase						      =					self.norm(_lowerCamelCase					)
										if self.use_linear_projection:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
										else:
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)

										for transformer_block in self.transformer_blocks:
													__lowercase						      =					transformer_block(_lowerCamelCase     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)

										if self.use_linear_projection:
													__lowercase						      =					self.proj_out(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										else:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_out(_lowerCamelCase					)

										__lowercase						      =					hidden_states + residual
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# The second linear layer needs to be called
										# net_2 for now to match the index of the Sequential layer
										__lowercase						      =					FlaxGEGLU(self.dim     ,						self.dropout     ,						self.dtype					)
										__lowercase						      =					nn.Dense(self.dim     ,						dtype=self.dtype					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase						      =					self.net_a(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					self.net_a(_lowerCamelCase					)
										return hidden_states





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					self.dim * 4
										__lowercase						      =					nn.Dense(inner_dim * 2     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Union[str, Any]:




										'''simple docstring'''
										__lowercase						      =					self.proj(_lowerCamelCase					)
										__lowercase      ,				__lowercase						      =					jnp.split(_lowerCamelCase     ,						2     ,						axis=2					)
										return self.dropout_layer(hidden_linear * nn.gelu(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)





