
'''simple docstring'''





import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation







def snake_case_    (				lowercase__      ,					lowercase__      ,					lowercase__ = False      ,					lowercase__ = 1      ,					lowercase__ = 1      ,					lowercase__ = 1_0_0_0_0      ,					):
  assert len(timesteps.shape		) == 1, "Timesteps should be a 1d-array"

  UpperCAmelCase__			:				Union[str, Any]												=       embedding_dim // 2
  UpperCAmelCase__			:				Optional[int]												=       -math.log(UpperCamelCase_		) * torch.arange(
      start=0      ,					end=UpperCamelCase_      ,					dtype=torch.floataa      ,					device=timesteps.device		)
  UpperCAmelCase__			:				Optional[Any]												=       exponent / (half_dim - downscale_freq_shift)

  UpperCAmelCase__			:				Union[str, Any]												=       torch.exp(UpperCamelCase_		)
  UpperCAmelCase__			:				Tuple												=       timesteps[:, None].float() * emb[None, :]

  # scale embeddings
  UpperCAmelCase__			:				Any												=       scale * emb

  # concat sine and cosine embeddings
  UpperCAmelCase__			:				Optional[Any]												=       torch.cat([torch.sin(UpperCamelCase_		), torch.cos(UpperCamelCase_		)]      ,					dim=-1		)

  # flip sine and cosine embeddings
  if flip_sin_to_cos:
    UpperCAmelCase__			:				Optional[Any]												=       torch.cat([emb[:, half_dim:], emb[:, :half_dim]]      ,					dim=-1		)

  # zero pad
  if embedding_dim % 2 == 1:
    UpperCAmelCase__			:				List[str]												=       torch.nn.functional.pad(UpperCamelCase_      ,					(0, 1, 0, 0)		)
  return emb







def snake_case_    (				lowercase__      ,					lowercase__      ,					lowercase__=False      ,					lowercase__=0		):
  UpperCAmelCase__			:				Optional[Any]												=       np.arange(UpperCamelCase_      ,					dtype=np.floataa		)
  UpperCAmelCase__			:				int												=       np.arange(UpperCamelCase_      ,					dtype=np.floataa		)
  UpperCAmelCase__			:				Any												=       np.meshgrid(UpperCamelCase_      ,					UpperCamelCase_		)  # here w goes first
  UpperCAmelCase__			:				Union[str, Any]												=       np.stack(UpperCamelCase_      ,					axis=0		)

  UpperCAmelCase__			:				Optional[int]												=       grid.reshape([2, 1, grid_size, grid_size]		)
  UpperCAmelCase__			:				Tuple												=       get_ad_sincos_pos_embed_from_grid(UpperCamelCase_      ,					UpperCamelCase_		)
  if cls_token and extra_tokens > 0:
    UpperCAmelCase__			:				List[str]												=       np.concatenate([np.zeros([extra_tokens, embed_dim]		), pos_embed]      ,					axis=0		)
  return pos_embed







def snake_case_    (				lowercase__      ,					lowercase__		):






  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"		)

  # use half of dimensions to encode grid_h
  UpperCAmelCase__			:				Optional[Any]												=       get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[0]		)  # (H*W, D/2)
  UpperCAmelCase__			:				Optional[Any]												=       get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[1]		)  # (H*W, D/2)

  UpperCAmelCase__			:				str												=       np.concatenate([emb_h, emb_w]      ,					axis=1		)  # (H*W, D)
  return emb







def snake_case_    (				lowercase__      ,					lowercase__		):






  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"		)

  UpperCAmelCase__			:				Optional[int]												=       np.arange(embed_dim // 2      ,					dtype=np.floataa		)
  omega /= embed_dim / 2.0
  UpperCAmelCase__			:				int												=       1.0 / 1_0_0_0_0**omega  # (D/2,)

  UpperCAmelCase__			:				List[str]												=       pos.reshape(-1		)  # (M,)
  UpperCAmelCase__			:				int												=       np.einsum("m,d->md"      ,					UpperCamelCase_      ,					UpperCamelCase_		)  # (M, D/2), outer product

  UpperCAmelCase__			:				Optional[int]												=       np.sin(UpperCamelCase_		)  # (M, D/2)
  UpperCAmelCase__			:				Tuple												=       np.cos(UpperCamelCase_		)  # (M, D/2)

  UpperCAmelCase__			:				Union[str, Any]												=       np.concatenate([emb_sin, emb_cos]      ,					axis=1		)  # (M, D)
  return emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[Any] ,							snake_case__		:     List[str]=2_24 ,							snake_case__		:     str=2_24 ,							snake_case__		:     List[Any]=16 ,							snake_case__		:     Union[str, Any]=3 ,							snake_case__		:     Optional[int]=7_68 ,							snake_case__		:     Dict=False ,							snake_case__		:     Tuple=True ,							snake_case__		:     Tuple=True ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Tuple												=       (height // patch_size) * (width // patch_size)
   UpperCAmelCase__			:				Dict												=       flatten
   UpperCAmelCase__			:				List[str]												=       layer_norm

   UpperCAmelCase__			:				Tuple												=       nn.Convad(
       __snake_case ,							__snake_case ,							kernel_size=(patch_size, patch_size) ,							stride=__snake_case ,							bias=__snake_case   )
   if layer_norm:
     UpperCAmelCase__			:				str												=       nn.LayerNorm(__snake_case ,							elementwise_affine=__snake_case ,							eps=1e-6   )
   else:
     UpperCAmelCase__			:				List[str]												=       None

   UpperCAmelCase__			:				Any												=       get_ad_sincos_pos_embed(__snake_case ,							int(num_patches**0.5   )   )
   self.register_buffer("pos_embed" ,							torch.from_numpy(__snake_case   ).float().unsqueeze(0   ) ,							persistent=__snake_case   )







 def        UpperCamelCase     (				self		:     int ,							snake_case__		:     Optional[Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       self.proj(__snake_case   )
   if self.flatten:
     UpperCAmelCase__			:				Dict												=       latent.flatten(2   ).transpose(1 ,							2   )  # BCHW -> BNC
   if self.layer_norm:
     UpperCAmelCase__			:				Tuple												=       self.norm(__snake_case   )
   return latent + self.pos_embed






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Any ,							snake_case__		:     List[Any] ,							snake_case__		:     Optional[int] ,							snake_case__		:     Any = "silu" ,							snake_case__		:     Any = None ,							snake_case__		:     str = None ,							snake_case__		:     int=None ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				List[str]												=       nn.Linear(__snake_case ,							__snake_case   )

   if cond_proj_dim is not None:
     UpperCAmelCase__			:				str												=       nn.Linear(__snake_case ,							__snake_case ,							bias=__snake_case   )
   else:
     UpperCAmelCase__			:				Union[str, Any]												=       None

   UpperCAmelCase__			:				Tuple												=       get_activation(__snake_case   )

   if out_dim is not None:
     UpperCAmelCase__			:				Optional[Any]												=       out_dim
   else:
     UpperCAmelCase__			:				int												=       time_embed_dim
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(__snake_case ,							__snake_case   )

   if post_act_fn is None:
     UpperCAmelCase__			:				Dict												=       None
   else:
     UpperCAmelCase__			:				Tuple												=       get_activation(__snake_case   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     Dict ,							snake_case__		:     Optional[int]=None   ):



   '''simple docstring'''




   if condition is not None:
     UpperCAmelCase__			:				List[Any]												=       sample + self.cond_proj(__snake_case   )
   UpperCAmelCase__			:				str												=       self.linear_a(__snake_case   )

   if self.act is not None:
     UpperCAmelCase__			:				Optional[int]												=       self.act(__snake_case   )

   UpperCAmelCase__			:				List[str]												=       self.linear_a(__snake_case   )

   if self.post_act is not None:
     UpperCAmelCase__			:				Optional[int]												=       self.post_act(__snake_case   )
   return sample






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     int ,							snake_case__		:     Tuple ,							snake_case__		:     int ,							snake_case__		:     Dict   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				int												=       num_channels
   UpperCAmelCase__			:				Any												=       flip_sin_to_cos
   UpperCAmelCase__			:				Optional[Any]												=       downscale_freq_shift







 def        UpperCamelCase     (				self		:     List[Any] ,							snake_case__		:     Dict   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       get_timestep_embedding(
       __snake_case ,							self.num_channels ,							flip_sin_to_cos=self.flip_sin_to_cos ,							downscale_freq_shift=self.downscale_freq_shift ,							)
   return t_emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Dict ,							snake_case__		:     Union[str, Any] = 2_56 ,							snake_case__		:     Any = 1.0 ,							snake_case__		:     Any=True ,							snake_case__		:     int=True ,							snake_case__		:     List[str]=False   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Dict												=       nn.Parameter(torch.randn(__snake_case   ) * scale ,							requires_grad=__snake_case   )
   UpperCAmelCase__			:				Optional[int]												=       log
   UpperCAmelCase__			:				Optional[int]												=       flip_sin_to_cos

   if set_W_to_weight:
     # to delete later
     UpperCAmelCase__			:				Optional[Any]												=       nn.Parameter(torch.randn(__snake_case   ) * scale ,							requires_grad=__snake_case   )

     UpperCAmelCase__			:				Any												=       self.W







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     List[Any]   ):



   '''simple docstring'''




   if self.log:
     UpperCAmelCase__			:				List[Any]												=       torch.log(__snake_case   )

   UpperCAmelCase__			:				str												=       x[:, None] * self.weight[None, :] * 2 * np.pi

   if self.flip_sin_to_cos:
     UpperCAmelCase__			:				str												=       torch.cat([torch.cos(__snake_case   ), torch.sin(__snake_case   )] ,							dim=-1   )
   else:
     UpperCAmelCase__			:				Tuple												=       torch.cat([torch.sin(__snake_case   ), torch.cos(__snake_case   )] ,							dim=-1   )
   return out






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[str] ,							snake_case__		:     Any ,							snake_case__		:     List[str] ,							snake_case__		:     int ,							snake_case__		:     Any ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Any												=       height
   UpperCAmelCase__			:				int												=       width
   UpperCAmelCase__			:				Optional[Any]												=       num_embed
   UpperCAmelCase__			:				List[str]												=       embed_dim

   UpperCAmelCase__			:				List[Any]												=       nn.Embedding(self.num_embed ,							__snake_case   )
   UpperCAmelCase__			:				Optional[int]												=       nn.Embedding(self.height ,							__snake_case   )
   UpperCAmelCase__			:				List[str]												=       nn.Embedding(self.width ,							__snake_case   )







 def        UpperCamelCase     (				self		:     Tuple ,							snake_case__		:     List[str]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[str]												=       self.emb(__snake_case   )

   UpperCAmelCase__			:				Any												=       self.height_emb(torch.arange(self.height ,							device=index.device   ).view(1 ,							self.height   )   )

   # 1 x H x D -> 1 x H x 1 x D
   UpperCAmelCase__			:				Dict												=       height_emb.unsqueeze(2   )

   UpperCAmelCase__			:				int												=       self.width_emb(torch.arange(self.width ,							device=index.device   ).view(1 ,							self.width   )   )

   # 1 x W x D -> 1 x 1 x W x D
   UpperCAmelCase__			:				Optional[int]												=       width_emb.unsqueeze(1   )

   UpperCAmelCase__			:				Dict												=       height_emb + width_emb

   # 1 x H x W x D -> 1 x L xD
   UpperCAmelCase__			:				Optional[int]												=       pos_emb.view(1 ,							self.height * self.width ,							-1   )

   UpperCAmelCase__			:				List[str]												=       emb + pos_emb[:, : emb.shape[1], :]

   return emb






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[Any] ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     Dict ,							snake_case__		:     Union[str, Any]   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Optional[Any]												=       dropout_prob > 0
   UpperCAmelCase__			:				Dict												=       nn.Embedding(num_classes + use_cfg_embedding ,							__snake_case   )
   UpperCAmelCase__			:				str												=       num_classes
   UpperCAmelCase__			:				Any												=       dropout_prob







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     Tuple ,							snake_case__		:     int=None   ):



   '''simple docstring'''




   if force_drop_ids is None:
     UpperCAmelCase__			:				Any												=       torch.rand(labels.shape[0] ,							device=labels.device   ) < self.dropout_prob
   else:
     UpperCAmelCase__			:				int												=       torch.tensor(force_drop_ids == 1   )
   UpperCAmelCase__			:				int												=       torch.where(__snake_case ,							self.num_classes ,							__snake_case   )
   return labels







 def        UpperCamelCase     (				self		:     int ,							snake_case__		:     Optional[Any] ,							snake_case__		:     Optional[int]=None   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       self.dropout_prob > 0
   if (self.training and use_dropout) or (force_drop_ids is not None):
     UpperCAmelCase__			:				str												=       self.token_drop(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				str												=       self.embedding_table(__snake_case   )
   return embeddings






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[str] ,							snake_case__		:     Optional[int] = 10_24 ,							snake_case__		:     Any = 7_68 ,							snake_case__		:     List[Any] = 7_68 ,							snake_case__		:     Union[str, Any] = 10 ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Tuple												=       num_image_text_embeds
   UpperCAmelCase__			:				Dict												=       nn.Linear(__snake_case ,							self.num_image_text_embeds * cross_attention_dim   )
   UpperCAmelCase__			:				Optional[Any]												=       nn.Linear(__snake_case ,							__snake_case   )







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     List[Any] ,							snake_case__		:     Dict   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Dict												=       text_embeds.shape[0]

   # image
   UpperCAmelCase__			:				Optional[int]												=       self.image_embeds(__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       image_text_embeds.reshape(__snake_case ,							self.num_image_text_embeds ,							-1   )

   # text
   UpperCAmelCase__			:				Tuple												=       self.text_proj(__snake_case   )

   return torch.cat([image_text_embeds, text_embeds] ,							dim=1   )






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[int] ,							snake_case__		:     List[Any] = 7_68 ,							snake_case__		:     Optional[Any] = 7_68 ,							snake_case__		:     Dict = 32 ,							):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				Any												=       num_image_text_embeds
   UpperCAmelCase__			:				List[str]												=       nn.Linear(__snake_case ,							self.num_image_text_embeds * cross_attention_dim   )
   UpperCAmelCase__			:				List[str]												=       nn.LayerNorm(__snake_case   )







 def        UpperCamelCase     (				self		:     Tuple ,							snake_case__		:     str   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Dict												=       image_embeds.shape[0]

   # image
   UpperCAmelCase__			:				Tuple												=       self.image_embeds(__snake_case   )
   UpperCAmelCase__			:				Dict												=       image_embeds.reshape(__snake_case ,							self.num_image_text_embeds ,							-1   )
   UpperCAmelCase__			:				Union[str, Any]												=       self.norm(__snake_case   )
   return image_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Any ,							snake_case__		:     List[Any] ,							snake_case__		:     Optional[int] ,							snake_case__		:     List[Any]=0.1   ):



   '''simple docstring'''




   super().__init__()

   UpperCAmelCase__			:				str												=       Timesteps(num_channels=2_56 ,							flip_sin_to_cos=__snake_case ,							downscale_freq_shift=1   )
   UpperCAmelCase__			:				Tuple												=       TimestepEmbedding(in_channels=2_56 ,							time_embed_dim=__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       LabelEmbedding(__snake_case ,							__snake_case ,							__snake_case   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     Union[str, Any] ,							snake_case__		:     Dict=None   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[str]												=       self.time_proj(__snake_case   )
   UpperCAmelCase__			:				Any												=       self.timestep_embedder(timesteps_proj.to(dtype=__snake_case   )   )  # (N, D)

   UpperCAmelCase__			:				Optional[int]												=       self.class_embedder(__snake_case   )  # (N, D)

   UpperCAmelCase__			:				Union[str, Any]												=       timesteps_emb + class_labels  # (N, D)

   return conditioning






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Any ,							snake_case__		:     List[Any] ,							snake_case__		:     List[Any] ,							snake_case__		:     List[str] = 64   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				List[str]												=       nn.LayerNorm(__snake_case   )
   UpperCAmelCase__			:				str												=       AttentionPooling(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				Optional[int]												=       nn.LayerNorm(__snake_case   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     int   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       self.norma(__snake_case   )
   UpperCAmelCase__			:				str												=       self.pool(__snake_case   )
   UpperCAmelCase__			:				Dict												=       self.proj(__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       self.norma(__snake_case   )
   return hidden_states






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     List[Any] ,							snake_case__		:     Any = 7_68 ,							snake_case__		:     Dict = 7_68 ,							snake_case__		:     List[str] = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Dict												=       nn.Linear(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				Optional[Any]												=       nn.LayerNorm(__snake_case   )
   UpperCAmelCase__			:				Tuple												=       nn.Linear(__snake_case ,							__snake_case   )







 def        UpperCamelCase     (				self		:     Dict ,							snake_case__		:     Optional[Any] ,							snake_case__		:     Union[str, Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[Any]												=       self.text_proj(__snake_case   )
   UpperCAmelCase__			:				Dict												=       self.text_norm(__snake_case   )

   # image
   UpperCAmelCase__			:				int												=       self.image_proj(__snake_case   )

   return time_image_embeds + time_text_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Dict ,							snake_case__		:     Any = 7_68 ,							snake_case__		:     Any = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       nn.LayerNorm(__snake_case   )







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     Optional[int]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       self.image_proj(__snake_case   )
   UpperCAmelCase__			:				Any												=       self.image_norm(__snake_case   )
   return time_image_embeds






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     Optional[Any] ,							snake_case__		:     int = 7_68 ,							snake_case__		:     Optional[int] = 15_36   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Any												=       nn.Linear(__snake_case ,							__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       nn.LayerNorm(__snake_case   )
   UpperCAmelCase__			:				Dict												=       nn.Sequential(
       nn.Convad(3 ,							16 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(16 ,							16 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(16 ,							32 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(32 ,							32 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(32 ,							96 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(96 ,							96 ,							3 ,							padding=1   ) ,							nn.SiLU() ,							nn.Convad(96 ,							2_56 ,							3 ,							padding=1 ,							stride=2   ) ,							nn.SiLU() ,							nn.Convad(2_56 ,							4 ,							3 ,							padding=1   ) ,							)







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     List[str] ,							snake_case__		:     Optional[Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				int												=       self.image_proj(__snake_case   )
   UpperCAmelCase__			:				Optional[Any]												=       self.image_norm(__snake_case   )
   UpperCAmelCase__			:				List[Any]												=       self.input_hint_block(__snake_case   )
   return time_image_embeds, hint






class        UpperCAmelCase_       (						nn.Module							):

 '''simple docstring'''







 def __init__(				self		:     int ,							snake_case__		:     int ,							snake_case__		:     List[Any] ,							snake_case__		:     Optional[int]=None   ):



   '''simple docstring'''




   super().__init__()
   UpperCAmelCase__			:				Optional[int]												=       dtype
   UpperCAmelCase__			:				Tuple												=       nn.Parameter(torch.randn(1 ,							__snake_case   ) / embed_dim**0.5   )
   UpperCAmelCase__			:				Optional[Any]												=       nn.Linear(__snake_case ,							__snake_case ,							dtype=self.dtype   )
   UpperCAmelCase__			:				List[Any]												=       nn.Linear(__snake_case ,							__snake_case ,							dtype=self.dtype   )
   UpperCAmelCase__			:				List[str]												=       nn.Linear(__snake_case ,							__snake_case ,							dtype=self.dtype   )
   UpperCAmelCase__			:				Optional[Any]												=       num_heads
   UpperCAmelCase__			:				Optional[Any]												=       embed_dim // self.num_heads







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     str   ):



   '''simple docstring'''




   UpperCAmelCase__					, UpperCAmelCase__					, UpperCAmelCase__			:				str												=       x.size()

   def shape(snake_case__		:     Union[str, Any]   ):
     # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
     UpperCAmelCase__			:				Any												=       x.view(__snake_case ,							-1 ,							self.num_heads ,							self.dim_per_head   )
     # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
     UpperCAmelCase__			:				int												=       x.transpose(1 ,							2   )
     # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
     UpperCAmelCase__			:				int												=       x.reshape(bs * self.num_heads ,							-1 ,							self.dim_per_head   )
     # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
     UpperCAmelCase__			:				Optional[int]												=       x.transpose(1 ,							2   )
     return x

   UpperCAmelCase__			:				Optional[int]												=       x.mean(dim=1 ,							keepdim=__snake_case   ) + self.positional_embedding.to(x.dtype   )
   UpperCAmelCase__			:				List[str]												=       torch.cat([class_token, x] ,							dim=1   )  # (bs, length+1, width)

   # (bs*n_heads, class_token_length, dim_per_head)
   UpperCAmelCase__			:				str												=       shape(self.q_proj(__snake_case   )   )
   # (bs*n_heads, length+class_token_length, dim_per_head)
   UpperCAmelCase__			:				Optional[Any]												=       shape(self.k_proj(__snake_case   )   )
   UpperCAmelCase__			:				Optional[Any]												=       shape(self.v_proj(__snake_case   )   )

   # (bs*n_heads, class_token_length, length+class_token_length):
   UpperCAmelCase__			:				Optional[int]												=       1 / math.sqrt(math.sqrt(self.dim_per_head   )   )
   UpperCAmelCase__			:				Dict												=       torch.einsum("bct,bcs->bts" ,							q * scale ,							k * scale   )  # More stable with f16 than dividing afterwards
   UpperCAmelCase__			:				Optional[int]												=       torch.softmax(weight.float() ,							dim=-1   ).type(weight.dtype   )

   # (bs*n_heads, dim_per_head, class_token_length)
   UpperCAmelCase__			:				Any												=       torch.einsum("bts,bcs->bct" ,							__snake_case ,							__snake_case   )

   # (bs, length+1, width)
   UpperCAmelCase__			:				Optional[Any]												=       a.reshape(__snake_case ,							-1 ,							1   ).transpose(1 ,							2   )

   return a[:, 0, :]  # cls_token





'''simple docstring'''





import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
      pass

if is_speech_available():
      from transformers import EncodecFeatureExtractor, MusicgenProcessor


SCREAMING_SNAKE_CASE    =					random.Random()







def snake_case_    (				lowercase__      ,					lowercase__=1.0      ,					lowercase__=None      ,					lowercase__=None		):
  if rng is None:
    UpperCAmelCase__			:				Optional[int]												=       global_rng

  UpperCAmelCase__			:				List[str]												=       []
  for batch_idx in range(shape[0]		):
    values.append([]		)
    for _ in range(shape[1]		):
      values[-1].append(rng.random() * scale		)

  return values






@require_torch
@require_sentencepiece
class        UpperCAmelCase_       (						unittest.TestCase							):

 '''simple docstring'''







 def        UpperCamelCase     (				self		:     Optional[int]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Tuple												=       "facebook/musicgen-small"
   UpperCAmelCase__			:				Union[str, Any]												=       tempfile.mkdtemp()







 def        UpperCamelCase     (				self		:     Union[str, Any] ,							**snake_case__		:     Dict   ):



   '''simple docstring'''




   return TaTokenizer.from_pretrained(self.checkpoint ,							**snake_case__   )







 def        UpperCamelCase     (				self		:     Optional[int] ,							**snake_case__		:     str   ):



   '''simple docstring'''




   return EncodecFeatureExtractor.from_pretrained(self.checkpoint ,							**snake_case__   )







 def        UpperCamelCase     (				self		:     List[str]   ):



   '''simple docstring'''




   shutil.rmtree(self.tmpdirname   )







 def        UpperCamelCase     (				self		:     Any   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Any												=       self.get_tokenizer()
   UpperCAmelCase__			:				Tuple												=       self.get_feature_extractor()

   UpperCAmelCase__			:				int												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   processor.save_pretrained(self.tmpdirname   )
   UpperCAmelCase__			:				Dict												=       MusicgenProcessor.from_pretrained(self.tmpdirname   )

   self.assertEqual(processor.tokenizer.get_vocab() ,							tokenizer.get_vocab()   )
   self.assertIsInstance(processor.tokenizer ,							snake_case__   )

   self.assertEqual(processor.feature_extractor.to_json_string() ,							feature_extractor.to_json_string()   )
   self.assertIsInstance(processor.feature_extractor ,							snake_case__   )







 def        UpperCamelCase     (				self		:     Tuple   ):



   '''simple docstring'''




   UpperCAmelCase__			:				int												=       MusicgenProcessor(tokenizer=self.get_tokenizer() ,							feature_extractor=self.get_feature_extractor()   )
   processor.save_pretrained(self.tmpdirname   )

   UpperCAmelCase__			:				List[Any]												=       self.get_tokenizer(bos_token="(BOS)" ,							eos_token="(EOS)"   )
   UpperCAmelCase__			:				int												=       self.get_feature_extractor(do_normalize=snake_case__ ,							padding_value=1.0   )

   UpperCAmelCase__			:				Optional[Any]												=       MusicgenProcessor.from_pretrained(
       self.tmpdirname ,							bos_token="(BOS)" ,							eos_token="(EOS)" ,							do_normalize=snake_case__ ,							padding_value=1.0   )

   self.assertEqual(processor.tokenizer.get_vocab() ,							tokenizer_add_kwargs.get_vocab()   )
   self.assertIsInstance(processor.tokenizer ,							snake_case__   )

   self.assertEqual(processor.feature_extractor.to_json_string() ,							feature_extractor_add_kwargs.to_json_string()   )
   self.assertIsInstance(processor.feature_extractor ,							snake_case__   )







 def        UpperCamelCase     (				self		:     List[Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				int												=       self.get_feature_extractor()
   UpperCAmelCase__			:				int												=       self.get_tokenizer()

   UpperCAmelCase__			:				List[str]												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   UpperCAmelCase__			:				Optional[Any]												=       floats_list((3, 10_00)   )

   UpperCAmelCase__			:				List[Any]												=       feature_extractor(snake_case__ ,							return_tensors="np"   )
   UpperCAmelCase__			:				List[Any]												=       processor(snake_case__ ,							return_tensors="np"   )

   for key in input_feat_extract.keys():
     self.assertAlmostEqual(input_feat_extract[key].sum() ,							input_processor[key].sum() ,							delta=1e-2   )







 def        UpperCamelCase     (				self		:     List[str]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       self.get_feature_extractor()
   UpperCAmelCase__			:				Union[str, Any]												=       self.get_tokenizer()

   UpperCAmelCase__			:				Optional[int]												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   UpperCAmelCase__			:				List[Any]												=       "This is a test string"

   UpperCAmelCase__			:				Dict												=       processor(text=snake_case__   )

   UpperCAmelCase__			:				Any												=       tokenizer(snake_case__   )

   for key in encoded_tok.keys():
     self.assertListEqual(encoded_tok[key] ,							encoded_processor[key]   )







 def        UpperCamelCase     (				self		:     int   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Union[str, Any]												=       self.get_feature_extractor()
   UpperCAmelCase__			:				str												=       self.get_tokenizer()

   UpperCAmelCase__			:				List[Any]												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   UpperCAmelCase__			:				Optional[int]												=       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

   UpperCAmelCase__			:				Any												=       processor.batch_decode(sequences=snake_case__   )
   UpperCAmelCase__			:				Optional[Any]												=       tokenizer.batch_decode(snake_case__   )

   self.assertListEqual(snake_case__ ,							snake_case__   )







 def        UpperCamelCase     (				self		:     Tuple   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Any												=       self.get_feature_extractor()
   UpperCAmelCase__			:				List[Any]												=       self.get_tokenizer()

   UpperCAmelCase__			:				Union[str, Any]												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   self.assertListEqual(
       processor.model_input_names ,							feature_extractor.model_input_names ,							msg="`processor` and `feature_extractor` model input names do not match" ,							)







 def        UpperCamelCase     (				self		:     Dict   ):



   '''simple docstring'''




   UpperCAmelCase__			:				int												=       self.get_feature_extractor(padding_side="left"   )
   UpperCAmelCase__			:				str												=       self.get_tokenizer()

   UpperCAmelCase__			:				int												=       MusicgenProcessor(tokenizer=snake_case__ ,							feature_extractor=snake_case__   )

   UpperCAmelCase__			:				List[Any]												=       [floats_list((1, x)   )[0] for x in range(5 ,							20 ,							5   )]
   UpperCAmelCase__			:				List[Any]												=       processor(snake_case__   ).padding_mask

   UpperCAmelCase__			:				Dict												=       np.asarray(floats_list((3, 20)   )   )[:, None, :]
   UpperCAmelCase__			:				Optional[Any]												=       processor.batch_decode(snake_case__ ,							padding_mask=snake_case__   )

   self.assertIsInstance(snake_case__ ,							snake_case__   )

   for audio in decoded_audios:
     self.assertIsInstance(snake_case__ ,							np.ndarray   )

   self.assertTrue(decoded_audios[0].shape == (1, 10)   )
   self.assertTrue(decoded_audios[1].shape == (1, 15)   )
   self.assertTrue(decoded_audios[2].shape == (1, 20)   )



