


'''simple docstring'''





import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp


def 					a__    (				UpperCamelCase_	:			Any,      UpperCamelCase_	:			Dict,      UpperCamelCase_	:			Tuple,      UpperCamelCase_	:			str,      UpperCamelCase_	:			int = 4_096	):
   UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :List[str]            =			key.shape[-3:]
   UpperCAmelCase__  :str            =			value.shape[-1]
   UpperCAmelCase__  :Optional[Any]            =			min(UpperCamelCase_,      UpperCamelCase_	)
   UpperCAmelCase__  :List[str]            =			query / jnp.sqrt(UpperCamelCase_	)

   @functools.partial(jax.checkpoint,      prevent_cse=UpperCamelCase_	)
   def summarize_chunk(UpperCamelCase_	:			int,      UpperCamelCase_	:			int,      UpperCamelCase_	:			Any	):
      UpperCAmelCase__  :Optional[int]            =			jnp.einsum('''...qhd,...khd->...qhk''',      UpperCamelCase_,      UpperCamelCase_,      precision=UpperCamelCase_	)

      UpperCAmelCase__  :Tuple            =			jnp.max(UpperCamelCase_,      axis=-1,      keepdims=UpperCamelCase_	)
      UpperCAmelCase__  :List[Any]            =			jax.lax.stop_gradient(UpperCamelCase_	)
      UpperCAmelCase__  :List[Any]            =			jnp.exp(attn_weights - max_score	)

      UpperCAmelCase__  :Tuple            =			jnp.einsum('''...vhf,...qhv->...qhf''',      UpperCamelCase_,      UpperCamelCase_,      precision=UpperCamelCase_	)
      UpperCAmelCase__  :Tuple            =			jnp.einsum('''...qhk->...qh''',      UpperCamelCase_	)

      return (exp_values, exp_weights.sum(axis=-1	), max_score)

   def chunk_scanner(UpperCamelCase_	:			int	):
      # julienne key array
      UpperCAmelCase__  :Optional[Any]            =			jax.lax.dynamic_slice(
          operand=UpperCamelCase_,      start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0],      slice_sizes=list(key.shape[:-3]	) + [key_chunk_size, num_heads, k_features],      )

      # julienne value array
      UpperCAmelCase__  :Optional[int]            =			jax.lax.dynamic_slice(
          operand=UpperCamelCase_,      start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0],      slice_sizes=list(value.shape[:-3]	) + [key_chunk_size, num_heads, v_features],      )

      return summarize_chunk(UpperCamelCase_,      UpperCamelCase_,      UpperCamelCase_	)

   UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :List[Any]            =			jax.lax.map(f=UpperCamelCase_,      xs=jnp.arange(0,      UpperCamelCase_,      UpperCamelCase_	)	)

   UpperCAmelCase__  :Optional[Any]            =			jnp.max(UpperCamelCase_,      axis=0,      keepdims=UpperCamelCase_	)
   UpperCAmelCase__  :str            =			jnp.exp(chunk_max - global_max	)

   chunk_values *= jnp.expand_dims(UpperCamelCase_,      axis=-1	)
   chunk_weights *= max_diffs

   UpperCAmelCase__  :List[str]            =			chunk_values.sum(axis=0	)
   UpperCAmelCase__  :Optional[Any]            =			jnp.expand_dims(UpperCamelCase_,      -1	).sum(axis=0	)

   return all_values / all_weights


def 					a__    (				UpperCamelCase_	:			Optional[int],      UpperCamelCase_	:			Dict,      UpperCamelCase_	:			Any,      UpperCamelCase_	:			Optional[Any]=jax.lax.Precision.HIGHEST,      UpperCamelCase_	:			int = 1_024,      UpperCamelCase_	:			int = 4_096	):
   UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :Any            =			query.shape[-3:]

   def chunk_scanner(UpperCamelCase_	:			Tuple,      UpperCamelCase_	:			Optional[int]	):
      # julienne query array
      UpperCAmelCase__  :Union[str, Any]            =			jax.lax.dynamic_slice(
          operand=UpperCamelCase_,      start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0],      slice_sizes=list(query.shape[:-3]	) + [min(UpperCamelCase_,      UpperCamelCase_	), num_heads, q_features],      )

      return (
          chunk_idx + query_chunk_size,  # unused ignore it
          _query_chunk_attention(
              query=UpperCamelCase_,      key=UpperCamelCase_,      value=UpperCamelCase_,      precision=UpperCamelCase_,      key_chunk_size=UpperCamelCase_	),
      )

   UpperCAmelCase__      ,	UpperCAmelCase__  :Optional[int]            =			jax.lax.scan(
       f=UpperCamelCase_,      init=0,      xs=UpperCamelCase_,      length=math.ceil(num_q / query_chunk_size	)	)

   return jnp.concatenate(UpperCamelCase_,      axis=-3	)  # fuse the chunked result back




class 							UpperCAmelCase ( nn.Module	):
 UpperCAmelCase				=				42
 UpperCAmelCase				=				8
 UpperCAmelCase				=				64
 UpperCAmelCase				=				0.0
 UpperCAmelCase				=				False
 UpperCAmelCase				=				jnp.floataa


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Tuple   ):
    UpperCAmelCase__  :Dict            =			self.dim_head * self.heads
    UpperCAmelCase__  :int            =			self.dim_head**-0.5

    # Weights were exported with old names {to_q, to_k, to_v, to_out}
    UpperCAmelCase__  :Optional[Any]            =			nn.Dense(__lowerCamelCase ,					use_bias=__lowerCamelCase ,					dtype=self.dtype ,					name='''to_q'''   )
    UpperCAmelCase__  :Any            =			nn.Dense(__lowerCamelCase ,					use_bias=__lowerCamelCase ,					dtype=self.dtype ,					name='''to_k'''   )
    UpperCAmelCase__  :Union[str, Any]            =			nn.Dense(__lowerCamelCase ,					use_bias=__lowerCamelCase ,					dtype=self.dtype ,					name='''to_v'''   )

    UpperCAmelCase__  :Optional[int]            =			nn.Dense(self.query_dim ,					dtype=self.dtype ,					name='''to_out_0'''   )
    UpperCAmelCase__  :Optional[int]            =			nn.Dropout(rate=self.dropout   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[Any] ,					__lowerCamelCase       :  Tuple   ):
    UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :List[str]            =			tensor.shape
    UpperCAmelCase__  :int            =			self.heads
    UpperCAmelCase__  :List[Any]            =			tensor.reshape(__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase ,					dim // head_size   )
    UpperCAmelCase__  :str            =			jnp.transpose(__lowerCamelCase ,					(0, 2, 1, 3)   )
    UpperCAmelCase__  :str            =			tensor.reshape(batch_size * head_size ,					__lowerCamelCase ,					dim // head_size   )
    return tensor


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any] ,					__lowerCamelCase       :  Dict   ):
    UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :Dict            =			tensor.shape
    UpperCAmelCase__  :int            =			self.heads
    UpperCAmelCase__  :int            =			tensor.reshape(batch_size // head_size ,					__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase   )
    UpperCAmelCase__  :Union[str, Any]            =			jnp.transpose(__lowerCamelCase ,					(0, 2, 1, 3)   )
    UpperCAmelCase__  :Tuple            =			tensor.reshape(batch_size // head_size ,					__lowerCamelCase ,					dim * head_size   )
    return tensor





 def __call__(			self       :  Optional[int] ,					__lowerCamelCase       :  List[str] ,					__lowerCamelCase       :  str=None ,					__lowerCamelCase       :  Optional[int]=True   ):
    UpperCAmelCase__  :Optional[Any]            =			hidden_states if context is None else context

    UpperCAmelCase__  :str            =			self.query(__lowerCamelCase   )
    UpperCAmelCase__  :str            =			self.key(__lowerCamelCase   )
    UpperCAmelCase__  :str            =			self.value(__lowerCamelCase   )

    UpperCAmelCase__  :Tuple            =			self.reshape_heads_to_batch_dim(__lowerCamelCase   )
    UpperCAmelCase__  :int            =			self.reshape_heads_to_batch_dim(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			self.reshape_heads_to_batch_dim(__lowerCamelCase   )

    if self.use_memory_efficient_attention:
       UpperCAmelCase__  :List[Any]            =			query_states.transpose(1 ,					0 ,					2   )
       UpperCAmelCase__  :int            =			key_states.transpose(1 ,					0 ,					2   )
       UpperCAmelCase__  :str            =			value_states.transpose(1 ,					0 ,					2   )

       # this if statement create a chunk size for each layer of the unet
       # the chunk size is equal to the query_length dimension of the deepest layer of the unet

       UpperCAmelCase__  :Tuple            =			query_states.shape[-3]
       if flatten_latent_dim % 6_4 == 0:
          UpperCAmelCase__  :List[Any]            =			int(flatten_latent_dim / 6_4   )
       elif flatten_latent_dim % 1_6 == 0:
          UpperCAmelCase__  :Optional[int]            =			int(flatten_latent_dim / 1_6   )
       elif flatten_latent_dim % 4 == 0:
          UpperCAmelCase__  :int            =			int(flatten_latent_dim / 4   )
       else:
          UpperCAmelCase__  :List[str]            =			int(__lowerCamelCase   )

       UpperCAmelCase__  :Tuple            =			jax_memory_efficient_attention(
           __lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase ,					query_chunk_size=__lowerCamelCase ,					key_chunk_size=4_0_9_6 * 4   )

       UpperCAmelCase__  :Tuple            =			hidden_states.transpose(1 ,					0 ,					2   )
    else:
       # compute attentions
       UpperCAmelCase__  :int            =			jnp.einsum('''b i d, b j d->b i j''' ,					__lowerCamelCase ,					__lowerCamelCase   )
       UpperCAmelCase__  :Tuple            =			attention_scores * self.scale
       UpperCAmelCase__  :Tuple            =			nn.softmax(__lowerCamelCase ,					axis=2   )

       # attend to values
       UpperCAmelCase__  :List[str]            =			jnp.einsum('''b i j, b j d -> b i d''' ,					__lowerCamelCase ,					__lowerCamelCase   )

    UpperCAmelCase__  :Optional[int]            =			self.reshape_batch_dim_to_heads(__lowerCamelCase   )
    UpperCAmelCase__  :int            =			self.proj_attn(__lowerCamelCase   )
    return self.dropout_layer(__lowerCamelCase ,					deterministic=__lowerCamelCase   )




class 							UpperCAmelCase ( nn.Module	):
 UpperCAmelCase				=				42
 UpperCAmelCase				=				42
 UpperCAmelCase				=				42
 UpperCAmelCase				=				0.0
 UpperCAmelCase				=				False
 UpperCAmelCase				=				jnp.floataa
 UpperCAmelCase				=				False


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Union[str, Any]   ):
    # self attention (or cross_attention if only_cross_attention is True)
    UpperCAmelCase__  :List[Any]            =			FlaxAttention(
        self.dim ,					self.n_heads ,					self.d_head ,					self.dropout ,					self.use_memory_efficient_attention ,					dtype=self.dtype   )
    # cross attention
    UpperCAmelCase__  :Optional[int]            =			FlaxAttention(
        self.dim ,					self.n_heads ,					self.d_head ,					self.dropout ,					self.use_memory_efficient_attention ,					dtype=self.dtype   )
    UpperCAmelCase__  :str            =			FlaxFeedForward(dim=self.dim ,					dropout=self.dropout ,					dtype=self.dtype   )
    UpperCAmelCase__  :Any            =			nn.LayerNorm(epsilon=1e-5 ,					dtype=self.dtype   )
    UpperCAmelCase__  :List[Any]            =			nn.LayerNorm(epsilon=1e-5 ,					dtype=self.dtype   )
    UpperCAmelCase__  :int            =			nn.LayerNorm(epsilon=1e-5 ,					dtype=self.dtype   )
    UpperCAmelCase__  :Tuple            =			nn.Dropout(rate=self.dropout   )





 def __call__(			self       :  Union[str, Any] ,					__lowerCamelCase       :  str ,					__lowerCamelCase       :  Optional[Any] ,					__lowerCamelCase       :  str=True   ):
    # self attention
    UpperCAmelCase__  :int            =			hidden_states
    if self.only_cross_attention:
       UpperCAmelCase__  :int            =			self.attna(self.norma(__lowerCamelCase   ) ,					__lowerCamelCase ,					deterministic=__lowerCamelCase   )
    else:
       UpperCAmelCase__  :Optional[int]            =			self.attna(self.norma(__lowerCamelCase   ) ,					deterministic=__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			hidden_states + residual

    # cross attention
    UpperCAmelCase__  :List[str]            =			hidden_states
    UpperCAmelCase__  :Optional[Any]            =			self.attna(self.norma(__lowerCamelCase   ) ,					__lowerCamelCase ,					deterministic=__lowerCamelCase   )
    UpperCAmelCase__  :List[Any]            =			hidden_states + residual

    # feed forward
    UpperCAmelCase__  :List[Any]            =			hidden_states
    UpperCAmelCase__  :Optional[Any]            =			self.ff(self.norma(__lowerCamelCase   ) ,					deterministic=__lowerCamelCase   )
    UpperCAmelCase__  :List[str]            =			hidden_states + residual

    return self.dropout_layer(__lowerCamelCase ,					deterministic=__lowerCamelCase   )




class 							UpperCAmelCase ( nn.Module	):
 UpperCAmelCase				=				42
 UpperCAmelCase				=				42
 UpperCAmelCase				=				42
 UpperCAmelCase				=				1
 UpperCAmelCase				=				0.0
 UpperCAmelCase				=				False
 UpperCAmelCase				=				False
 UpperCAmelCase				=				jnp.floataa
 UpperCAmelCase				=				False


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Any   ):
    UpperCAmelCase__  :Optional[Any]            =			nn.GroupNorm(num_groups=3_2 ,					epsilon=1e-5   )

    UpperCAmelCase__  :Optional[Any]            =			self.n_heads * self.d_head
    if self.use_linear_projection:
       UpperCAmelCase__  :Union[str, Any]            =			nn.Dense(__lowerCamelCase ,					dtype=self.dtype   )
    else:
       UpperCAmelCase__  :Optional[Any]            =			nn.Conv(
           __lowerCamelCase ,					kernel_size=(1, 1) ,					strides=(1, 1) ,					padding='''VALID''' ,					dtype=self.dtype ,					)

    UpperCAmelCase__  :Union[str, Any]            =			[
        FlaxBasicTransformerBlock(
            __lowerCamelCase ,					self.n_heads ,					self.d_head ,					dropout=self.dropout ,					only_cross_attention=self.only_cross_attention ,					dtype=self.dtype ,					use_memory_efficient_attention=self.use_memory_efficient_attention ,					)
        for _ in range(self.depth   )
    ]

    if self.use_linear_projection:
       UpperCAmelCase__  :int            =			nn.Dense(__lowerCamelCase ,					dtype=self.dtype   )
    else:
       UpperCAmelCase__  :str            =			nn.Conv(
           __lowerCamelCase ,					kernel_size=(1, 1) ,					strides=(1, 1) ,					padding='''VALID''' ,					dtype=self.dtype ,					)

    UpperCAmelCase__  :int            =			nn.Dropout(rate=self.dropout   )





 def __call__(			self       :  Any ,					__lowerCamelCase       :  Optional[int] ,					__lowerCamelCase       :  Optional[int] ,					__lowerCamelCase       :  Tuple=True   ):
    UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__      ,	UpperCAmelCase__  :str            =			hidden_states.shape
    UpperCAmelCase__  :Tuple            =			hidden_states
    UpperCAmelCase__  :str            =			self.norm(__lowerCamelCase   )
    if self.use_linear_projection:
       UpperCAmelCase__  :Dict            =			hidden_states.reshape(__lowerCamelCase ,					height * width ,					__lowerCamelCase   )
       UpperCAmelCase__  :Any            =			self.proj_in(__lowerCamelCase   )
    else:
       UpperCAmelCase__  :Union[str, Any]            =			self.proj_in(__lowerCamelCase   )
       UpperCAmelCase__  :Tuple            =			hidden_states.reshape(__lowerCamelCase ,					height * width ,					__lowerCamelCase   )

    for transformer_block in self.transformer_blocks:
       UpperCAmelCase__  :Optional[int]            =			transformer_block(__lowerCamelCase ,					__lowerCamelCase ,					deterministic=__lowerCamelCase   )

    if self.use_linear_projection:
       UpperCAmelCase__  :Optional[Any]            =			self.proj_out(__lowerCamelCase   )
       UpperCAmelCase__  :int            =			hidden_states.reshape(__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase   )
    else:
       UpperCAmelCase__  :Optional[Any]            =			hidden_states.reshape(__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase ,					__lowerCamelCase   )
       UpperCAmelCase__  :Dict            =			self.proj_out(__lowerCamelCase   )

    UpperCAmelCase__  :int            =			hidden_states + residual
    return self.dropout_layer(__lowerCamelCase ,					deterministic=__lowerCamelCase   )




class 							UpperCAmelCase ( nn.Module	):
 UpperCAmelCase				=				42
 UpperCAmelCase				=				0.0
 UpperCAmelCase				=				jnp.floataa


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Tuple   ):
    # The second linear layer needs to be called
    # net_2 for now to match the index of the Sequential layer
    UpperCAmelCase__  :Tuple            =			FlaxGEGLU(self.dim ,					self.dropout ,					self.dtype   )
    UpperCAmelCase__  :Optional[Any]            =			nn.Dense(self.dim ,					dtype=self.dtype   )





 def __call__(			self       :  int ,					__lowerCamelCase       :  Optional[int] ,					__lowerCamelCase       :  Optional[int]=True   ):
    UpperCAmelCase__  :Union[str, Any]            =			self.net_a(__lowerCamelCase ,					deterministic=__lowerCamelCase   )
    UpperCAmelCase__  :Any            =			self.net_a(__lowerCamelCase   )
    return hidden_states




class 							UpperCAmelCase ( nn.Module	):
 UpperCAmelCase				=				42
 UpperCAmelCase				=				0.0
 UpperCAmelCase				=				jnp.floataa


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[Any]   ):
    UpperCAmelCase__  :Dict            =			self.dim * 4
    UpperCAmelCase__  :Any            =			nn.Dense(inner_dim * 2 ,					dtype=self.dtype   )
    UpperCAmelCase__  :List[Any]            =			nn.Dropout(rate=self.dropout   )





 def __call__(			self       :  List[Any] ,					__lowerCamelCase       :  Optional[Any] ,					__lowerCamelCase       :  Dict=True   ):
    UpperCAmelCase__  :Union[str, Any]            =			self.proj(__lowerCamelCase   )
    UpperCAmelCase__      ,	UpperCAmelCase__  :Tuple            =			jnp.split(__lowerCamelCase ,					2 ,					axis=2   )
    return self.dropout_layer(hidden_linear * nn.gelu(__lowerCamelCase   ) ,					deterministic=__lowerCamelCase   )







'''simple docstring'''





import heapq


def 					a__    (				UpperCamelCase_	:			Dict,      UpperCamelCase_	:			Tuple,      UpperCamelCase_	:			Dict	):
   UpperCAmelCase__  :List[Any]            =			[(0, start)]  # cost from start node,end node
   UpperCAmelCase__  :Optional[int]            =			set()
   while heap:
      ((UpperCAmelCase__)      ,	(UpperCAmelCase__))  :Optional[Any]            =			heapq.heappop(UpperCamelCase_	)
      if u in visited:
         continue
      visited.add(UpperCamelCase_	)
      if u == end:
         return cost
      for v, c in graph[u]:
         if v in visited:
            continue
         UpperCAmelCase__  :int            =			cost + c
         heapq.heappush(UpperCamelCase_,      (next_item, v)	)
   return -1


__lowerCamelCase       				=	{
    '''A''': [['''B''', 2], ['''C''', 5]],
    '''B''': [['''A''', 2], ['''D''', 3], ['''E''', 1], ['''F''', 1]],
    '''C''': [['''A''', 5], ['''F''', 3]],
    '''D''': [['''B''', 3]],
    '''E''': [['''B''', 4], ['''F''', 3]],
    '''F''': [['''C''', 3], ['''E''', 3]],
}
__lowerCamelCase       				=	{
    '''B''': [['''C''', 1]],
    '''C''': [['''D''', 1]],
    '''D''': [['''F''', 1]],
    '''E''': [['''B''', 1], ['''F''', 3]],
    '''F''': [],
}
__lowerCamelCase       				=	{
    '''B''': [['''C''', 1]],
    '''C''': [['''D''', 1]],
    '''D''': [['''F''', 1]],
    '''E''': [['''B''', 1], ['''G''', 2]],
    '''F''': [],
    '''G''': [['''F''', 1]],
}

__lowerCamelCase       				=	dijkstra(G, '''E''', '''C''')
print(short_distance)  # E -- 3 --> F -- 3 --> C == 6

__lowerCamelCase       				=	dijkstra(Ga, '''E''', '''F''')
print(short_distance)  # E -- 3 --> F == 3

__lowerCamelCase       				=	dijkstra(Ga, '''E''', '''F''')
print(short_distance)  # E -- 2 --> G -- 1 --> F == 3

if __name__ == "__main__":
    import doctest

    doctest.testmod()



