



'''simple docstring'''





import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation







def snake_case_    (				lowercase__      ,					lowercase__      ,					lowercase__ = False      ,					lowercase__ = 1      ,					lowercase__ = 1      ,					lowercase__ = 1_0_0_0_0      ,					):
  assert len(timesteps.shape		) == 1, "Timesteps should be a 1d-array"

  UpperCAmelCase__			:				int												=       embedding_dim // 2
  UpperCAmelCase__			:				Tuple												=       -math.log(lowercase__		) * torch.arange(
      start=0      ,					end=lowercase__      ,					dtype=torch.floataa      ,					device=timesteps.device		)
  UpperCAmelCase__			:				List[Any]												=       exponent / (half_dim - downscale_freq_shift)

  UpperCAmelCase__			:				List[Any]												=       torch.exp(lowercase__		)
  UpperCAmelCase__			:				List[str]												=       timesteps[:, None].float() * emb[None, :]

  # scale embeddings
  UpperCAmelCase__			:				str												=       scale * emb

  # concat sine and cosine embeddings
  UpperCAmelCase__			:				Optional[int]												=       torch.cat([torch.sin(lowercase__		), torch.cos(lowercase__		)]      ,					dim=-1		)

  # flip sine and cosine embeddings
  if flip_sin_to_cos:
    UpperCAmelCase__			:				str												=       torch.cat([emb[:, half_dim:], emb[:, :half_dim]]      ,					dim=-1		)

  # zero pad
  if embedding_dim % 2 == 1:
    UpperCAmelCase__			:				Any												=       torch.nn.functional.pad(lowercase__      ,					(0, 1, 0, 0)		)
  return emb







def snake_case_    (				lowercase__      ,					lowercase__      ,					lowercase__=False      ,					lowercase__=0		):
  UpperCAmelCase__			:				Any												=       np.arange(lowercase__      ,					dtype=np.floataa		)
  UpperCAmelCase__			:				Dict												=       np.arange(lowercase__      ,					dtype=np.floataa		)
  UpperCAmelCase__			:				Dict												=       np.meshgrid(lowercase__      ,					lowercase__		)  # here w goes first
  UpperCAmelCase__			:				Optional[Any]												=       np.stack(lowercase__      ,					axis=0		)

  UpperCAmelCase__			:				Optional[int]												=       grid.reshape([2, 1, grid_size, grid_size]		)
  UpperCAmelCase__			:				List[str]												=       get_ad_sincos_pos_embed_from_grid(lowercase__      ,					lowercase__		)
  if cls_token and extra_tokens > 0:
    UpperCAmelCase__			:				Any												=       np.concatenate([np.zeros([extra_tokens, embed_dim]		), pos_embed]      ,					axis=0		)
  return pos_embed







def snake_case_    (				lowercase__      ,					lowercase__		):
  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"		)

  # use half of dimensions to encode grid_h
  UpperCAmelCase__			:				str												=       get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[0]		)  # (H*W, D/2)
  UpperCAmelCase__			:				Union[str, Any]												=       get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[1]		)  # (H*W, D/2)

  UpperCAmelCase__			:				int												=       np.concatenate([emb_h, emb_w]      ,					axis=1		)  # (H*W, D)
  return emb







def snake_case_    (				lowercase__      ,					lowercase__		):
  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"		)

  UpperCAmelCase__			:				List[Any]												=       np.arange(embed_dim // 2      ,					dtype=np.floataa		)
  omega /= embed_dim / 2.0
  UpperCAmelCase__			:				Dict												=       1.0 / 1_0_0_0_0**omega  # (D/2,)

  UpperCAmelCase__			:				int												=       pos.reshape(-1		)  # (M,)
  UpperCAmelCase__			:				Union[str, Any]												=       np.einsum("m,d->md"      ,					lowercase__      ,					lowercase__		)  # (M, D/2), outer product

  UpperCAmelCase__			:				Optional[int]												=       np.sin(lowercase__		)  # (M, D/2)
  UpperCAmelCase__			:				Optional[int]												=       np.cos(lowercase__		)  # (M, D/2)

  UpperCAmelCase__			:				Union[str, Any]												=       np.concatenate([emb_sin, emb_cos]      ,					axis=1		)  # (M, D)
  return emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Dict ,							snake_case__		:     Optional[Any]=2_24 ,							snake_case__		:     Dict=2_24 ,							snake_case__		:     int=16 ,							snake_case__		:     List[str]=3 ,							snake_case__		:     Optional[Any]=7_68 ,							snake_case__		:     Dict=False ,							snake_case__		:     List[Any]=True ,							snake_case__		:     Optional[Any]=True ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Optional[int]												=       (height // patch_size) * (width // patch_size)
   UpperCAmelCase__			:				Tuple												=       flatten
   UpperCAmelCase__			:				Dict												=       layer_norm

   UpperCAmelCase__			:				Tuple												=       nn.Convad(
       snake_case__ ,							snake_case__ ,							kernel_size=(patch_size, patch_size) ,							stride=snake_case__ ,							bias=snake_case__   )
   if layer_norm:
     UpperCAmelCase__			:				Tuple												=       nn.LayerNorm(snake_case__ ,							elementwise_affine=snake_case__ ,							eps=1e-6   )
   else:
     UpperCAmelCase__			:				str												=       None

   UpperCAmelCase__			:				Any												=       get_ad_sincos_pos_embed(snake_case__ ,							int(num_patches**0.5   )   )
   self.register_buffer("pos_embed" ,							torch.from_numpy(snake_case__   ).float().unsqueeze(0   ) ,							persistent=snake_case__   )







 def        UpperCamelCase     (				self		:     Union[str, Any] ,							snake_case__		:     int   ):



   '''simple docstring'''




   UpperCAmelCase__			:				int												=       self.proj(snake_case__   )
   if self.flatten:
     UpperCAmelCase__			:				int												=       latent.flatten(2   ).transpose(1 ,							2   )  # BCHW -> BNC
   if self.layer_norm:
     UpperCAmelCase__			:				Any												=       self.norm(snake_case__   )
   return latent + self.pos_embed






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[str] ,							snake_case__		:     int ,							snake_case__		:     int ,							snake_case__		:     str = "silu" ,							snake_case__		:     int = None ,							snake_case__		:     Optional[str] = None ,							snake_case__		:     List[str]=None ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Optional[int]												=       nn.Linear(snake_case__ ,							snake_case__   )

   if cond_proj_dim is not None:
     UpperCAmelCase__			:				List[str]												=       nn.Linear(snake_case__ ,							snake_case__ ,							bias=snake_case__   )
   else:
     UpperCAmelCase__			:				List[Any]												=       None

   UpperCAmelCase__			:				Tuple												=       get_activation(snake_case__   )

   if out_dim is not None:
     UpperCAmelCase__			:				Dict												=       out_dim
   else:
     UpperCAmelCase__			:				Optional[int]												=       time_embed_dim
   UpperCAmelCase__			:				Optional[Any]												=       nn.Linear(snake_case__ ,							snake_case__   )

   if post_act_fn is None:
     UpperCAmelCase__			:				str												=       None
   else:
     UpperCAmelCase__			:				Dict												=       get_activation(snake_case__   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     str=None   ):



   '''simple docstring'''




   if condition is not None:
     UpperCAmelCase__			:				Any												=       sample + self.cond_proj(snake_case__   )
   UpperCAmelCase__			:				List[Any]												=       self.linear_a(snake_case__   )

   if self.act is not None:
     UpperCAmelCase__			:				Optional[Any]												=       self.act(snake_case__   )

   UpperCAmelCase__			:				str												=       self.linear_a(snake_case__   )

   if self.post_act is not None:
     UpperCAmelCase__			:				Optional[int]												=       self.post_act(snake_case__   )
   return sample






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     str ,							snake_case__		:     int ,							snake_case__		:     bool ,							snake_case__		:     float   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Any												=       num_channels
   UpperCAmelCase__			:				Optional[Any]												=       flip_sin_to_cos
   UpperCAmelCase__			:				Tuple												=       downscale_freq_shift







 def        UpperCamelCase     (				self		:     List[Any] ,							snake_case__		:     Tuple   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[Any]												=       get_timestep_embedding(
       snake_case__ ,							self.num_channels ,							flip_sin_to_cos=self.flip_sin_to_cos ,							downscale_freq_shift=self.downscale_freq_shift ,							)
   return t_emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     str ,							snake_case__		:     int = 2_56 ,							snake_case__		:     float = 1.0 ,							snake_case__		:     List[str]=True ,							snake_case__		:     Dict=True ,							snake_case__		:     str=False   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Optional[int]												=       nn.Parameter(torch.randn(snake_case__   ) * scale ,							requires_grad=snake_case__   )
   UpperCAmelCase__			:				Union[str, Any]												=       log
   UpperCAmelCase__			:				Union[str, Any]												=       flip_sin_to_cos

   if set_W_to_weight:
     # to delete later
     UpperCAmelCase__			:				int												=       nn.Parameter(torch.randn(snake_case__   ) * scale ,							requires_grad=snake_case__   )

     UpperCAmelCase__			:				Tuple												=       self.W







 def        UpperCamelCase     (				self		:     Optional[Any] ,							snake_case__		:     Union[str, Any]   ):



   '''simple docstring'''




   if self.log:
     UpperCAmelCase__			:				Any												=       torch.log(snake_case__   )

   UpperCAmelCase__			:				int												=       x[:, None] * self.weight[None, :] * 2 * np.pi

   if self.flip_sin_to_cos:
     UpperCAmelCase__			:				str												=       torch.cat([torch.cos(snake_case__   ), torch.sin(snake_case__   )] ,							dim=-1   )
   else:
     UpperCAmelCase__			:				List[str]												=       torch.cat([torch.sin(snake_case__   ), torch.cos(snake_case__   )] ,							dim=-1   )
   return out






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[Any] ,							snake_case__		:     int ,							snake_case__		:     int ,							snake_case__		:     int ,							snake_case__		:     int ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				List[str]												=       height
   UpperCAmelCase__			:				Union[str, Any]												=       width
   UpperCAmelCase__			:				Optional[int]												=       num_embed
   UpperCAmelCase__			:				List[Any]												=       embed_dim

   UpperCAmelCase__			:				List[str]												=       nn.Embedding(self.num_embed ,							snake_case__   )
   UpperCAmelCase__			:				str												=       nn.Embedding(self.height ,							snake_case__   )
   UpperCAmelCase__			:				Union[str, Any]												=       nn.Embedding(self.width ,							snake_case__   )







 def        UpperCamelCase     (				self		:     int ,							snake_case__		:     str   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[str]												=       self.emb(snake_case__   )

   UpperCAmelCase__			:				Optional[Any]												=       self.height_emb(torch.arange(self.height ,							device=index.device   ).view(1 ,							self.height   )   )

   # 1 x H x D -> 1 x H x 1 x D
   UpperCAmelCase__			:				Dict												=       height_emb.unsqueeze(2   )

   UpperCAmelCase__			:				Dict												=       self.width_emb(torch.arange(self.width ,							device=index.device   ).view(1 ,							self.width   )   )

   # 1 x W x D -> 1 x 1 x W x D
   UpperCAmelCase__			:				int												=       width_emb.unsqueeze(1   )

   UpperCAmelCase__			:				Tuple												=       height_emb + width_emb

   # 1 x H x W x D -> 1 x L xD
   UpperCAmelCase__			:				Any												=       pos_emb.view(1 ,							self.height * self.width ,							-1   )

   UpperCAmelCase__			:				Union[str, Any]												=       emb + pos_emb[:, : emb.shape[1], :]

   return emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[int] ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     Tuple ,							snake_case__		:     Optional[int]   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Optional[int]												=       dropout_prob > 0
   UpperCAmelCase__			:				Union[str, Any]												=       nn.Embedding(num_classes + use_cfg_embedding ,							snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       num_classes
   UpperCAmelCase__			:				Optional[int]												=       dropout_prob







 def        UpperCamelCase     (				self		:     Dict ,							snake_case__		:     Tuple ,							snake_case__		:     Union[str, Any]=None   ):



   '''simple docstring'''




   if force_drop_ids is None:
     UpperCAmelCase__			:				Optional[Any]												=       torch.rand(labels.shape[0] ,							device=labels.device   ) < self.dropout_prob
   else:
     UpperCAmelCase__			:				Dict												=       torch.tensor(force_drop_ids == 1   )
   UpperCAmelCase__			:				List[Any]												=       torch.where(snake_case__ ,							self.num_classes ,							snake_case__   )
   return labels







 def        UpperCamelCase     (				self		:     List[str] ,							snake_case__		:     torch.LongTensor ,							snake_case__		:     Dict=None   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[str]												=       self.dropout_prob > 0
   if (self.training and use_dropout) or (force_drop_ids is not None):
     UpperCAmelCase__			:				Optional[Any]												=       self.token_drop(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				str												=       self.embedding_table(snake_case__   )
   return embeddings






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[Any] ,							snake_case__		:     int = 10_24 ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 10 ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Union[str, Any]												=       num_image_text_embeds
   UpperCAmelCase__			:				Dict												=       nn.Linear(snake_case__ ,							self.num_image_text_embeds * cross_attention_dim   )
   UpperCAmelCase__			:				Any												=       nn.Linear(snake_case__ ,							snake_case__   )







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     torch.FloatTensor ,							snake_case__		:     torch.FloatTensor   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[Any]												=       text_embeds.shape[0]

   # image
   UpperCAmelCase__			:				Optional[int]												=       self.image_embeds(snake_case__   )
   UpperCAmelCase__			:				str												=       image_text_embeds.reshape(snake_case__ ,							self.num_image_text_embeds ,							-1   )

   # text
   UpperCAmelCase__			:				int												=       self.text_proj(snake_case__   )

   return torch.cat([image_text_embeds, text_embeds] ,							dim=1   )






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[str] ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 32 ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				int												=       num_image_text_embeds
   UpperCAmelCase__			:				int												=       nn.Linear(snake_case__ ,							self.num_image_text_embeds * cross_attention_dim   )
   UpperCAmelCase__			:				List[str]												=       nn.LayerNorm(snake_case__   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     torch.FloatTensor   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       image_embeds.shape[0]

   # image
   UpperCAmelCase__			:				str												=       self.image_embeds(snake_case__   )
   UpperCAmelCase__			:				List[str]												=       image_embeds.reshape(snake_case__ ,							self.num_image_text_embeds ,							-1   )
   UpperCAmelCase__			:				List[Any]												=       self.norm(snake_case__   )
   return image_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Union[str, Any] ,							snake_case__		:     Dict ,							snake_case__		:     List[Any] ,							snake_case__		:     int=0.1   ):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Optional[int]												=       Timesteps(num_channels=2_56 ,							flip_sin_to_cos=snake_case__ ,							downscale_freq_shift=1   )
   UpperCAmelCase__			:				Dict												=       TimestepEmbedding(in_channels=2_56 ,							time_embed_dim=snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       LabelEmbedding(snake_case__ ,							snake_case__ ,							snake_case__   )







 def        UpperCamelCase     (				self		:     Union[str, Any] ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     Dict ,							snake_case__		:     int=None   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       self.time_proj(snake_case__   )
   UpperCAmelCase__			:				int												=       self.timestep_embedder(timesteps_proj.to(dtype=snake_case__   )   )  # (N, D)

   UpperCAmelCase__			:				Tuple												=       self.class_embedder(snake_case__   )  # (N, D)

   UpperCAmelCase__			:				str												=       timesteps_emb + class_labels  # (N, D)

   return conditioning






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[Any] ,							snake_case__		:     int ,							snake_case__		:     int ,							snake_case__		:     int = 64   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Tuple												=       nn.LayerNorm(snake_case__   )
   UpperCAmelCase__			:				List[Any]												=       AttentionPooling(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				List[str]												=       nn.Linear(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				Tuple												=       nn.LayerNorm(snake_case__   )







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     Tuple   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Dict												=       self.norma(snake_case__   )
   UpperCAmelCase__			:				List[Any]												=       self.pool(snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       self.proj(snake_case__   )
   UpperCAmelCase__			:				List[str]												=       self.norma(snake_case__   )
   return hidden_states






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     int ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				str												=       nn.LayerNorm(snake_case__   )
   UpperCAmelCase__			:				List[str]												=       nn.Linear(snake_case__ ,							snake_case__   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     torch.FloatTensor ,							snake_case__		:     torch.FloatTensor   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       self.text_proj(snake_case__   )
   UpperCAmelCase__			:				List[Any]												=       self.text_norm(snake_case__   )

   # image
   UpperCAmelCase__			:				Optional[int]												=       self.image_proj(snake_case__   )

   return time_image_embeds + time_text_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Dict ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Union[str, Any]												=       nn.Linear(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       nn.LayerNorm(snake_case__   )







 def        UpperCamelCase     (				self		:     Optional[int] ,							snake_case__		:     torch.FloatTensor   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       self.image_proj(snake_case__   )
   UpperCAmelCase__			:				Optional[Any]												=       self.image_norm(snake_case__   )
   return time_image_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Tuple ,							snake_case__		:     int = 7_68 ,							snake_case__		:     int = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				str												=       nn.Linear(snake_case__ ,							snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       nn.LayerNorm(snake_case__   )
   UpperCAmelCase__			:				Any												=       nn.Sequential(
       nn.Convad(3 ,							16 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(16 ,							16 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(16 ,							32 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(32 ,							32 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(32 ,							96 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(96 ,							96 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(96 ,							2_56 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(2_56 ,							4 ,							3 ,							padding=1   ) ,							)







 def        UpperCamelCase     (				self		:     Optional[Any] ,							snake_case__		:     torch.FloatTensor ,							snake_case__		:     torch.FloatTensor   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[Any]												=       self.image_proj(snake_case__   )
   UpperCAmelCase__			:				Tuple												=       self.image_norm(snake_case__   )
   UpperCAmelCase__			:				int												=       self.input_hint_block(snake_case__   )
   return time_image_embeds, hint






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[Any] ,							snake_case__		:     Tuple ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     List[str]=None   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Optional[Any]												=       dtype
   UpperCAmelCase__			:				Any												=       nn.Parameter(torch.randn(1 ,							snake_case__   ) / embed_dim**0.5   )
   UpperCAmelCase__			:				Any												=       nn.Linear(snake_case__ ,							snake_case__ ,							dtype=self.dtype   )
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(snake_case__ ,							snake_case__ ,							dtype=self.dtype   )
   UpperCAmelCase__			:				Optional[Any]												=       nn.Linear(snake_case__ ,							snake_case__ ,							dtype=self.dtype   )
   UpperCAmelCase__			:				int												=       num_heads
   UpperCAmelCase__			:				str												=       embed_dim // self.num_heads







 def        UpperCamelCase     (				self		:     Optional[Any] ,							snake_case__		:     List[Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[str]												=       x.size()

   def shape(snake_case__		:     List[Any]   ):
     # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
     UpperCAmelCase__			:				Dict												=       x.view(snake_case__ ,							-1 ,							self.num_heads ,							self.dim_per_head   )
     # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
     UpperCAmelCase__			:				Any												=       x.transpose(1 ,							2   )
     # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
     UpperCAmelCase__			:				Union[str, Any]												=       x.reshape(bs * self.num_heads ,							-1 ,							self.dim_per_head   )
     # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
     UpperCAmelCase__			:				List[Any]												=       x.transpose(1 ,							2   )
     return x

   UpperCAmelCase__			:				Optional[Any]												=       x.mean(dim=1 ,							keepdim=snake_case__   ) + self.positional_embedding.to(x.dtype   )
   UpperCAmelCase__			:				Optional[Any]												=       torch.cat([class_token, x] ,							dim=1   )  # (bs, length+1, width)

   # (bs*n_heads, class_token_length, dim_per_head)
   UpperCAmelCase__			:				Union[str, Any]												=       shape(self.q_proj(snake_case__   )   )
   # (bs*n_heads, length+class_token_length, dim_per_head)
   UpperCAmelCase__			:				Optional[int]												=       shape(self.k_proj(snake_case__   )   )
   UpperCAmelCase__			:				Optional[Any]												=       shape(self.v_proj(snake_case__   )   )

   # (bs*n_heads, class_token_length, length+class_token_length):
   UpperCAmelCase__			:				Optional[Any]												=       1 / math.sqrt(math.sqrt(self.dim_per_head   )   )
   UpperCAmelCase__			:				Dict												=       torch.einsum("bct,bcs->bts" ,							q * scale ,							k * scale   )  # More stable with f16 than dividing afterwards
   UpperCAmelCase__			:				Optional[int]												=       torch.softmax(weight.float() ,							dim=-1   ).type(weight.dtype   )

   # (bs*n_heads, dim_per_head, class_token_length)
   UpperCAmelCase__			:				int												=       torch.einsum("bts,bcs->bct" ,							snake_case__ ,							snake_case__   )

   # (bs, length+1, width)
   UpperCAmelCase__			:				Union[str, Any]												=       a.reshape(snake_case__ ,							-1 ,							1   ).transpose(1 ,							2   )

   return a[:, 0, :]  # cls_token





'''simple docstring'''





def snake_case_    (				lowercase__      ,					lowercase__ = 1_3		):
  UpperCAmelCase__			:				Optional[Any]												=       ""
  for c in s:
    if "A" <= c <= "Z":
      out += chr(ord("A"		) + (ord(lowercase__		) - ord("A"		) + n) % 2_6		)
    elif "a" <= c <= "z":
      out += chr(ord("a"		) + (ord(lowercase__		) - ord("a"		) + n) % 2_6		)
    else:
      out += c
  return out







def snake_case_    (				):
  UpperCAmelCase__			:				Dict												=       input("Enter message: "		)

  UpperCAmelCase__			:				Optional[int]												=       dencrypt(lowercase__      ,					1_3		)
  print("Encryption:"      ,					lowercase__		)

  UpperCAmelCase__			:				Union[str, Any]												=       dencrypt(lowercase__      ,					1_3		)
  print("Decryption: "      ,					lowercase__		)


if __name__ == "__main__":
      import doctest

      doctest.testmod()
      main()



