




'''simple docstring'''



import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation







def 			_UpperCamelCase (_lowerCamelCase							:     torch.Tensor   ,	_lowerCamelCase							:     int   ,	_lowerCamelCase							:     bool = False   ,	_lowerCamelCase							:     float = 1   ,	_lowerCamelCase							:     float = 1   ,	_lowerCamelCase							:     int = 1_00_00   ,	)->   List[Any]:




     '''simple docstring'''





     assert len(timesteps.shape	) == 1, "Timesteps should be a 1d-array"

     __snake_case    					=  embedding_dim // 2
     __snake_case    					=  -math.log(_lowerCamelCase	) * torch.arange(
         start=0   ,	end=_lowerCamelCase   ,	dtype=torch.floataa   ,	device=timesteps.device	)
     __snake_case    					=  exponent / (half_dim - downscale_freq_shift)

     __snake_case    					=  torch.exp(_lowerCamelCase	)
     __snake_case    					=  timesteps[:, None].float() * emb[None, :]

     # scale embeddings
     __snake_case    					=  scale * emb

     # concat sine and cosine embeddings
     __snake_case    					=  torch.cat([torch.sin(_lowerCamelCase	), torch.cos(_lowerCamelCase	)]   ,	dim=-1	)

     # flip sine and cosine embeddings
     if flip_sin_to_cos:
          __snake_case    					=  torch.cat([emb[:, half_dim:], emb[:, :half_dim]]   ,	dim=-1	)

     # zero pad
     if embedding_dim % 2 == 1:
          __snake_case    					=  torch.nn.functional.pad(_lowerCamelCase   ,	(0, 1, 0, 0)	)
     return emb







def 			_UpperCamelCase (_lowerCamelCase							:     str   ,	_lowerCamelCase							:     Any   ,	_lowerCamelCase							:     Dict=False   ,	_lowerCamelCase							:     Union[str, Any]=0	)->   List[str]:




     '''simple docstring'''





     __snake_case    					=  np.arange(_lowerCamelCase   ,	dtype=np.floataa	)
     __snake_case    					=  np.arange(_lowerCamelCase   ,	dtype=np.floataa	)
     __snake_case    					=  np.meshgrid(_lowerCamelCase   ,	_lowerCamelCase	)  # here w goes first
     __snake_case    					=  np.stack(_lowerCamelCase   ,	axis=0	)

     __snake_case    					=  grid.reshape([2, 1, grid_size, grid_size]	)
     __snake_case    					=  get_ad_sincos_pos_embed_from_grid(_lowerCamelCase   ,	_lowerCamelCase	)
     if cls_token and extra_tokens > 0:
          __snake_case    					=  np.concatenate([np.zeros([extra_tokens, embed_dim]	), pos_embed]   ,	axis=0	)
     return pos_embed







def 			_UpperCamelCase (_lowerCamelCase							:     int   ,	_lowerCamelCase							:     str	)->   Any:




     '''simple docstring'''





     if embed_dim % 2 != 0:
          raise ValueError('''embed_dim must be divisible by 2'''	)

     # use half of dimensions to encode grid_h
     __snake_case    					=  get_ad_sincos_pos_embed_from_grid(embed_dim // 2   ,	grid[0]	)  # (H*W, D/2)
     __snake_case    					=  get_ad_sincos_pos_embed_from_grid(embed_dim // 2   ,	grid[1]	)  # (H*W, D/2)

     __snake_case    					=  np.concatenate([emb_h, emb_w]   ,	axis=1	)  # (H*W, D)
     return emb







def 			_UpperCamelCase (_lowerCamelCase							:     Any   ,	_lowerCamelCase							:     List[Any]	)->   str:




     '''simple docstring'''





     if embed_dim % 2 != 0:
          raise ValueError('''embed_dim must be divisible by 2'''	)

     __snake_case    					=  np.arange(embed_dim // 2   ,	dtype=np.floataa	)
     omega /= embed_dim / 2.0
     __snake_case    					=  1.0 / 1_00_00**omega  # (D/2,)

     __snake_case    					=  pos.reshape(-1	)  # (M,)
     __snake_case    					=  np.einsum('''m,d->md'''   ,	_lowerCamelCase   ,	_lowerCamelCase	)  # (M, D/2), outer product

     __snake_case    					=  np.sin(_lowerCamelCase	)  # (M, D/2)
     __snake_case    					=  np.cos(_lowerCamelCase	)  # (M, D/2)

     __snake_case    					=  np.concatenate([emb_sin, emb_cos]   ,	axis=1	)  # (M, D)
     return emb





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE=224    ,  __SCREAMING_SNAKE_CASE=224    ,  __SCREAMING_SNAKE_CASE=16    ,  __SCREAMING_SNAKE_CASE=3    ,  __SCREAMING_SNAKE_CASE=768    ,  __SCREAMING_SNAKE_CASE=False    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=True    ,  )      ->  Optional[int]:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  (height // patch_size) * (width // patch_size)
            __snake_case    					=  flatten
            __snake_case    					=  layer_norm

            __snake_case    					=  nn.Convad(
                __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  kernel_size=(patch_size, patch_size)    ,  stride=__SCREAMING_SNAKE_CASE    ,  bias=__SCREAMING_SNAKE_CASE  )
            if layer_norm:
                 __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE    ,  elementwise_affine=__SCREAMING_SNAKE_CASE    ,  eps=1E-6  )
            else:
                 __snake_case    					=  None

            __snake_case    					=  get_ad_sincos_pos_embed(__SCREAMING_SNAKE_CASE    ,  int(num_patches**0.5  )  )
            self.register_buffer('''pos_embed'''    ,  torch.from_numpy(__SCREAMING_SNAKE_CASE  ).float().unsqueeze(0  )    ,  persistent=__SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            __snake_case    					=  self.proj(__SCREAMING_SNAKE_CASE  )
            if self.flatten:
                 __snake_case    					=  latent.flatten(2  ).transpose(1    ,  2  )  # BCHW -> BNC
            if self.layer_norm:
                 __snake_case    					=  self.norm(__SCREAMING_SNAKE_CASE  )
            return latent + self.pos_embed





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = "silu"    ,  __SCREAMING_SNAKE_CASE = None    ,  __SCREAMING_SNAKE_CASE = None    ,  __SCREAMING_SNAKE_CASE=None    ,  )      ->  List[str]:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )

            if cond_proj_dim is not None:
                 __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  bias=__SCREAMING_SNAKE_CASE  )
            else:
                 __snake_case    					=  None

            __snake_case    					=  get_activation(__SCREAMING_SNAKE_CASE  )

            if out_dim is not None:
                 __snake_case    					=  out_dim
            else:
                 __snake_case    					=  time_embed_dim
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )

            if post_act_fn is None:
                 __snake_case    					=  None
            else:
                 __snake_case    					=  get_activation(__SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  List[str]:



            '''simple docstring'''




            if condition is not None:
                 __snake_case    					=  sample + self.cond_proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.linear_a(__SCREAMING_SNAKE_CASE  )

            if self.act is not None:
                 __snake_case    					=  self.act(__SCREAMING_SNAKE_CASE  )

            __snake_case    					=  self.linear_a(__SCREAMING_SNAKE_CASE  )

            if self.post_act is not None:
                 __snake_case    					=  self.post_act(__SCREAMING_SNAKE_CASE  )
            return sample





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  num_channels
            __snake_case    					=  flip_sin_to_cos
            __snake_case    					=  downscale_freq_shift




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  get_timestep_embedding(
                __SCREAMING_SNAKE_CASE    ,  self.num_channels    ,  flip_sin_to_cos=self.flip_sin_to_cos    ,  downscale_freq_shift=self.downscale_freq_shift    ,  )
            return t_emb





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 256    ,  __SCREAMING_SNAKE_CASE = 1.0    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=False  )      ->  Optional[int]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  nn.Parameter(torch.randn(__SCREAMING_SNAKE_CASE  ) * scale    ,  requires_grad=__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  log
            __snake_case    					=  flip_sin_to_cos

            if set_W_to_weight:
                 # to delete later
                 __snake_case    					=  nn.Parameter(torch.randn(__SCREAMING_SNAKE_CASE  ) * scale    ,  requires_grad=__SCREAMING_SNAKE_CASE  )

                 __snake_case    					=  self.W




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if self.log:
                 __snake_case    					=  torch.log(__SCREAMING_SNAKE_CASE  )

            __snake_case    					=  x[:, None] * self.weight[None, :] * 2 * np.pi

            if self.flip_sin_to_cos:
                 __snake_case    					=  torch.cat([torch.cos(__SCREAMING_SNAKE_CASE  ), torch.sin(__SCREAMING_SNAKE_CASE  )]    ,  dim=-1  )
            else:
                 __snake_case    					=  torch.cat([torch.sin(__SCREAMING_SNAKE_CASE  ), torch.cos(__SCREAMING_SNAKE_CASE  )]    ,  dim=-1  )
            return out





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  )      ->  int:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  height
            __snake_case    					=  width
            __snake_case    					=  num_embed
            __snake_case    					=  embed_dim

            __snake_case    					=  nn.Embedding(self.num_embed    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.Embedding(self.height    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.Embedding(self.width    ,  __SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Dict:



            '''simple docstring'''




            __snake_case    					=  self.emb(__SCREAMING_SNAKE_CASE  )

            __snake_case    					=  self.height_emb(torch.arange(self.height    ,  device=index.device  ).view(1    ,  self.height  )  )

            # 1 x H x D -> 1 x H x 1 x D
            __snake_case    					=  height_emb.unsqueeze(2  )

            __snake_case    					=  self.width_emb(torch.arange(self.width    ,  device=index.device  ).view(1    ,  self.width  )  )

            # 1 x W x D -> 1 x 1 x W x D
            __snake_case    					=  width_emb.unsqueeze(1  )

            __snake_case    					=  height_emb + width_emb

            # 1 x H x W x D -> 1 x L xD
            __snake_case    					=  pos_emb.view(1    ,  self.height * self.width    ,  -1  )

            __snake_case    					=  emb + pos_emb[:, : emb.shape[1], :]

            return emb





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  dropout_prob > 0
            __snake_case    					=  nn.Embedding(num_classes + use_cfg_embedding    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  num_classes
            __snake_case    					=  dropout_prob


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  Optional[int]:



            '''simple docstring'''




            if force_drop_ids is None:
                 __snake_case    					=  torch.rand(labels.shape[0]    ,  device=labels.device  ) < self.dropout_prob
            else:
                 __snake_case    					=  torch.tensor(force_drop_ids == 1  )
            __snake_case    					=  torch.where(__SCREAMING_SNAKE_CASE    ,  self.num_classes    ,  __SCREAMING_SNAKE_CASE  )
            return labels




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  Union[str, Any]:



            '''simple docstring'''




            __snake_case    					=  self.dropout_prob > 0
            if (self.training and use_dropout) or (force_drop_ids is not None):
                 __snake_case    					=  self.token_drop(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.embedding_table(__SCREAMING_SNAKE_CASE  )
            return embeddings





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 1024    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 10    ,  )      ->  Any:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  num_image_text_embeds
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  self.num_image_text_embeds * cross_attention_dim  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  text_embeds.shape[0]

            # image
            __snake_case    					=  self.image_embeds(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  image_text_embeds.reshape(__SCREAMING_SNAKE_CASE    ,  self.num_image_text_embeds    ,  -1  )

            # text
            __snake_case    					=  self.text_proj(__SCREAMING_SNAKE_CASE  )

            return torch.cat([image_text_embeds, text_embeds]    ,  dim=1  )





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 32    ,  )      ->  List[str]:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  num_image_text_embeds
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  self.num_image_text_embeds * cross_attention_dim  )
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            __snake_case    					=  image_embeds.shape[0]

            # image
            __snake_case    					=  self.image_embeds(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  image_embeds.reshape(__SCREAMING_SNAKE_CASE    ,  self.num_image_text_embeds    ,  -1  )
            __snake_case    					=  self.norm(__SCREAMING_SNAKE_CASE  )
            return image_embeds





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=0.1  )      ->  Dict:



            '''simple docstring'''




            super().__init__()

            __snake_case    					=  Timesteps(num_channels=256    ,  flip_sin_to_cos=__SCREAMING_SNAKE_CASE    ,  downscale_freq_shift=1  )
            __snake_case    					=  TimestepEmbedding(in_channels=256    ,  time_embed_dim=__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  LabelEmbedding(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  List[str]:



            '''simple docstring'''




            __snake_case    					=  self.time_proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.timestep_embedder(timesteps_proj.to(dtype=__SCREAMING_SNAKE_CASE  )  )  # (N, D)

            __snake_case    					=  self.class_embedder(__SCREAMING_SNAKE_CASE  )  # (N, D)

            __snake_case    					=  timesteps_emb + class_labels  # (N, D)

            return conditioning





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = 64  )      ->  Union[str, Any]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  AttentionPooling(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  self.norma(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.pool(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.norma(__SCREAMING_SNAKE_CASE  )
            return hidden_states





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 1536  )      ->  Optional[int]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  int:



            '''simple docstring'''




            __snake_case    					=  self.text_proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.text_norm(__SCREAMING_SNAKE_CASE  )

            # image
            __snake_case    					=  self.image_proj(__SCREAMING_SNAKE_CASE  )

            return time_image_embeds + time_text_embeds





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 1536  )      ->  Tuple:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            __snake_case    					=  self.image_proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.image_norm(__SCREAMING_SNAKE_CASE  )
            return time_image_embeds





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE = 768    ,  __SCREAMING_SNAKE_CASE = 1536  )      ->  Union[str, Any]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.LayerNorm(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  nn.Sequential(
                nn.Convad(3    ,  16    ,  3    ,  padding=1  )    ,  nn.SiLU()    ,  nn.Convad(16    ,  16    ,  3    ,  padding=1  )    ,  nn.SiLU()    ,  nn.Convad(16    ,  32    ,  3    ,  padding=1    ,  stride=2  )    ,  nn.SiLU()    ,  nn.Convad(32    ,  32    ,  3    ,  padding=1  )    ,  nn.SiLU()    ,  nn.Convad(32    ,  96    ,  3    ,  padding=1    ,  stride=2  )    ,  nn.SiLU()    ,  nn.Convad(96    ,  96    ,  3    ,  padding=1  )    ,  nn.SiLU()    ,  nn.Convad(96    ,  256    ,  3    ,  padding=1    ,  stride=2  )    ,  nn.SiLU()    ,  nn.Convad(256    ,  4    ,  3    ,  padding=1  )    ,  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  str:



            '''simple docstring'''




            __snake_case    					=  self.image_proj(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.image_norm(__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  self.input_hint_block(__SCREAMING_SNAKE_CASE  )
            return time_image_embeds, hint





class     lowerCAmelCase   ( nn.Module):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  List[Any]:



            '''simple docstring'''




            super().__init__()
            __snake_case    					=  dtype
            __snake_case    					=  nn.Parameter(torch.randn(1    ,  __SCREAMING_SNAKE_CASE  ) / embed_dim**0.5  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  dtype=self.dtype  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  dtype=self.dtype  )
            __snake_case    					=  nn.Linear(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  dtype=self.dtype  )
            __snake_case    					=  num_heads
            __snake_case    					=  embed_dim // self.num_heads




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Dict:



            '''simple docstring'''




            __snake_case					,    __snake_case					,    __snake_case    					=  x.size()

            def shape(__SCREAMING_SNAKE_CASE  ):
                 # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
                 __snake_case    					=  x.view(__SCREAMING_SNAKE_CASE    ,  -1    ,  self.num_heads    ,  self.dim_per_head  )
                 # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
                 __snake_case    					=  x.transpose(1    ,  2  )
                 # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
                 __snake_case    					=  x.reshape(bs * self.num_heads    ,  -1    ,  self.dim_per_head  )
                 # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
                 __snake_case    					=  x.transpose(1    ,  2  )
                 return x

            __snake_case    					=  x.mean(dim=1    ,  keepdim=__SCREAMING_SNAKE_CASE  ) + self.positional_embedding.to(x.dtype  )
            __snake_case    					=  torch.cat([class_token, x]    ,  dim=1  )  # (bs, length+1, width)

            # (bs*n_heads, class_token_length, dim_per_head)
            __snake_case    					=  shape(self.q_proj(__SCREAMING_SNAKE_CASE  )  )
            # (bs*n_heads, length+class_token_length, dim_per_head)
            __snake_case    					=  shape(self.k_proj(__SCREAMING_SNAKE_CASE  )  )
            __snake_case    					=  shape(self.v_proj(__SCREAMING_SNAKE_CASE  )  )

            # (bs*n_heads, class_token_length, length+class_token_length):
            __snake_case    					=  1 / math.sqrt(math.sqrt(self.dim_per_head  )  )
            __snake_case    					=  torch.einsum('''bct,bcs->bts'''    ,  q * scale    ,  k * scale  )  # More stable with f16 than dividing afterwards
            __snake_case    					=  torch.softmax(weight.float()    ,  dim=-1  ).type(weight.dtype  )

            # (bs*n_heads, dim_per_head, class_token_length)
            __snake_case    					=  torch.einsum('''bts,bcs->bct'''    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )

            # (bs, length+1, width)
            __snake_case    					=  a.reshape(__SCREAMING_SNAKE_CASE    ,  -1    ,  1  ).transpose(1    ,  2  )

            return a[:, 0, :]  # cls_token












'''simple docstring'''



from ..utils import deprecate
from .attention_processor import (  # noqa: F401
    Attention,
    AttentionProcessor,
    AttnAddedKVProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRALinearLayer,
    LoRAXFormersAttnProcessor,
    SlicedAttnAddedKVProcessor,
    SlicedAttnProcessor,
    XFormersAttnProcessor,
)
from .attention_processor import AttnProcessor as AttnProcessorRename  # noqa: F401


deprecate(
    '''cross_attention''',
    '''0.20.0''',
    '''Importing from cross_attention is deprecated. Please import from diffusers.models.attention_processor instead.''',
    standard_warn=False,
)


UpperCAmelCase_       :  Optional[int]      =		AttentionProcessor





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Tuple:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Tuple:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  int:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  List[str]:



            '''simple docstring'''




            __snake_case    					=  F'''{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {"".join(self.__class__.__name__.split("Cross"  )  )} instead.'''
            deprecate('''cross_attention'''    ,  '''0.20.0'''    ,  __SCREAMING_SNAKE_CASE    ,  standard_warn=__SCREAMING_SNAKE_CASE  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )






