





'''simple docstring'''


import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation
def 							UpperCAmelCase_						(		__lowerCamelCase      :      torch.Tensor			,__lowerCamelCase      :      int			,__lowerCamelCase      :      bool = False			,__lowerCamelCase      :      float = 1			,__lowerCamelCase      :      float = 1			,__lowerCamelCase      :      int = 1_00_00			,):
   assert len(timesteps.shape				) == 1, "Timesteps should be a 1d-array"

   lowercase_    :Tuple					    =				embedding_dim // 2
   lowercase_    :List[Any]					    =				-math.log(__lowerCamelCase				) * torch.arange(
       start=0			,end=__lowerCamelCase			,dtype=torch.floataa			,device=timesteps.device				)
   lowercase_    :List[Any]					    =				exponent / (half_dim - downscale_freq_shift)

   lowercase_    :Optional[Any]					    =				torch.exp(__lowerCamelCase				)
   lowercase_    :List[str]					    =				timesteps[:, None].float() * emb[None, :]

   # scale embeddings
   lowercase_    :Any					    =				scale * emb

   # concat sine and cosine embeddings
   lowercase_    :List[str]					    =				torch.cat([torch.sin(__lowerCamelCase				), torch.cos(__lowerCamelCase				)]			,dim=-1				)

   # flip sine and cosine embeddings
   if flip_sin_to_cos:
      lowercase_    :List[str]					    =				torch.cat([emb[:, half_dim:], emb[:, :half_dim]]			,dim=-1				)

   # zero pad
   if embedding_dim % 2 == 1:
      lowercase_    :Union[str, Any]					    =				torch.nn.functional.pad(__lowerCamelCase			,(0, 1, 0, 0)				)
   return emb
def 							UpperCAmelCase_						(		__lowerCamelCase      :      List[str]			,__lowerCamelCase      :      Optional[Any]			,__lowerCamelCase      :      Dict=False			,__lowerCamelCase      :      List[Any]=0				):
   lowercase_    :int					    =				np.arange(__lowerCamelCase			,dtype=np.floataa				)
   lowercase_    :Any					    =				np.arange(__lowerCamelCase			,dtype=np.floataa				)
   lowercase_    :str					    =				np.meshgrid(__lowerCamelCase			,__lowerCamelCase				)  # here w goes first
   lowercase_    :List[str]					    =				np.stack(__lowerCamelCase			,axis=0				)

   lowercase_    :Tuple					    =				grid.reshape([2, 1, grid_size, grid_size]				)
   lowercase_    :Optional[Any]					    =				get_ad_sincos_pos_embed_from_grid(__lowerCamelCase			,__lowerCamelCase				)
   if cls_token and extra_tokens > 0:
      lowercase_    :Optional[int]					    =				np.concatenate([np.zeros([extra_tokens, embed_dim]				), pos_embed]			,axis=0				)
   return pos_embed
def 							UpperCAmelCase_						(		__lowerCamelCase      :      List[str]			,__lowerCamelCase      :      Union[str, Any]				):
   if embed_dim % 2 != 0:
      raise ValueError("embed_dim must be divisible by 2"				)

   # use half of dimensions to encode grid_h
   lowercase_    :Any					    =				get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,grid[0]				)  # (H*W, D/2)
   lowercase_    :List[Any]					    =				get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,grid[1]				)  # (H*W, D/2)

   lowercase_    :List[str]					    =				np.concatenate([emb_h, emb_w]			,axis=1				)  # (H*W, D)
   return emb
def 							UpperCAmelCase_						(		__lowerCamelCase      :      Dict			,__lowerCamelCase      :      Union[str, Any]				):
   if embed_dim % 2 != 0:
      raise ValueError("embed_dim must be divisible by 2"				)

   lowercase_    :Tuple					    =				np.arange(embed_dim // 2			,dtype=np.floataa				)
   omega /= embed_dim / 2.0
   lowercase_    :Optional[Any]					    =				1.0 / 1_00_00**omega  # (D/2,)

   lowercase_    :int					    =				pos.reshape(-1				)  # (M,)
   lowercase_    :Union[str, Any]					    =				np.einsum("m,d->md"			,__lowerCamelCase			,__lowerCamelCase				)  # (M, D/2), outer product

   lowercase_    :Any					    =				np.sin(__lowerCamelCase				)  # (M, D/2)
   lowercase_    :Optional[Any]					    =				np.cos(__lowerCamelCase				)  # (M, D/2)

   lowercase_    :Any					    =				np.concatenate([emb_sin, emb_cos]			,axis=1				)  # (M, D)
   return emb
class    a_     (       nn.Module							):





     def __init__(				self	:      Union[str, Any]			,   lowercase	:      Any=224			,   lowercase	:      Dict=224			,   lowercase	:      Dict=16			,   lowercase	:      Union[str, Any]=3			,   lowercase	:      str=768			,   lowercase	:      Dict=False			,   lowercase	:      str=True			,   lowercase	:      Tuple=True			,   ):





        """simple docstring"""




        super().__init__()

        lowercase_    :int					    =				(height // patch_size) * (width // patch_size)
        lowercase_    :Optional[Any]					    =				flatten
        lowercase_    :Dict					    =				layer_norm

        lowercase_    :Tuple					    =				nn.Convad(
            lowercase			,   lowercase			,   kernel_size=(patch_size, patch_size)			,   stride=lowercase			,   bias=lowercase		)
        if layer_norm:
           lowercase_    :int					    =				nn.LayerNorm(lowercase			,   elementwise_affine=lowercase			,   eps=1e-6		)
        else:
           lowercase_    :Dict					    =				None

        lowercase_    :Optional[int]					    =				get_ad_sincos_pos_embed(lowercase			,   int(num_patches**0.5		)		)
        self.register_buffer("pos_embed"			,   torch.from_numpy(lowercase		).float().unsqueeze(0		)			,   persistent=lowercase		)







     def 							lowercase__   (				self	:      List[Any]			,   lowercase	:      Dict		):





        """simple docstring"""




        lowercase_    :List[str]					    =				self.proj(lowercase		)
        if self.flatten:
           lowercase_    :Optional[Any]					    =				latent.flatten(2		).transpose(1			,   2		)  # BCHW -> BNC
        if self.layer_norm:
           lowercase_    :List[str]					    =				self.norm(lowercase		)
        return latent + self.pos_embed







class    a_     (       nn.Module							):





     def __init__(				self	:      Union[str, Any]			,   lowercase	:      int			,   lowercase	:      int			,   lowercase	:      str = "silu"			,   lowercase	:      int = None			,   lowercase	:      Optional[str] = None			,   lowercase	:      Optional[int]=None			,   ):





        """simple docstring"""




        super().__init__()

        lowercase_    :List[str]					    =				nn.Linear(lowercase			,   lowercase		)

        if cond_proj_dim is not None:
           lowercase_    :Union[str, Any]					    =				nn.Linear(lowercase			,   lowercase			,   bias=lowercase		)
        else:
           lowercase_    :Any					    =				None

        lowercase_    :List[str]					    =				get_activation(lowercase		)

        if out_dim is not None:
           lowercase_    :Dict					    =				out_dim
        else:
           lowercase_    :Union[str, Any]					    =				time_embed_dim
        lowercase_    :List[Any]					    =				nn.Linear(lowercase			,   lowercase		)

        if post_act_fn is None:
           lowercase_    :List[Any]					    =				None
        else:
           lowercase_    :Any					    =				get_activation(lowercase		)







     def 							lowercase__   (				self	:      Any			,   lowercase	:      List[str]			,   lowercase	:      Any=None		):





        """simple docstring"""




        if condition is not None:
           lowercase_    :List[Any]					    =				sample + self.cond_proj(lowercase		)
        lowercase_    :Any					    =				self.linear_a(lowercase		)

        if self.act is not None:
           lowercase_    :Optional[int]					    =				self.act(lowercase		)

        lowercase_    :Tuple					    =				self.linear_a(lowercase		)

        if self.post_act is not None:
           lowercase_    :Optional[int]					    =				self.post_act(lowercase		)
        return sample







class    a_     (       nn.Module							):





     def __init__(				self	:      Optional[int]			,   lowercase	:      int			,   lowercase	:      bool			,   lowercase	:      float		):





        """simple docstring"""




        super().__init__()
        lowercase_    :int					    =				num_channels
        lowercase_    :Optional[int]					    =				flip_sin_to_cos
        lowercase_    :str					    =				downscale_freq_shift







     def 							lowercase__   (				self	:      str			,   lowercase	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :int					    =				get_timestep_embedding(
            lowercase			,   self.num_channels			,   flip_sin_to_cos=self.flip_sin_to_cos			,   downscale_freq_shift=self.downscale_freq_shift			,   )
        return t_emb







class    a_     (       nn.Module							):





     def __init__(				self	:      str			,   lowercase	:      int = 256			,   lowercase	:      float = 1.0			,   lowercase	:      Union[str, Any]=True			,   lowercase	:      Optional[int]=True			,   lowercase	:      List[Any]=False		):





        """simple docstring"""




        super().__init__()
        lowercase_    :Dict					    =				nn.Parameter(torch.randn(lowercase		) * scale			,   requires_grad=lowercase		)
        lowercase_    :List[Any]					    =				log
        lowercase_    :str					    =				flip_sin_to_cos

        if set_W_to_weight:
           # to delete later
           lowercase_    :Dict					    =				nn.Parameter(torch.randn(lowercase		) * scale			,   requires_grad=lowercase		)

           lowercase_    :int					    =				self.W







     def 							lowercase__   (				self	:      Dict			,   lowercase	:      Any		):





        """simple docstring"""




        if self.log:
           lowercase_    :int					    =				torch.log(lowercase		)

        lowercase_    :Union[str, Any]					    =				x[:, None] * self.weight[None, :] * 2 * np.pi

        if self.flip_sin_to_cos:
           lowercase_    :Optional[Any]					    =				torch.cat([torch.cos(lowercase		), torch.sin(lowercase		)]			,   dim=-1		)
        else:
           lowercase_    :Tuple					    =				torch.cat([torch.sin(lowercase		), torch.cos(lowercase		)]			,   dim=-1		)
        return out







class    a_     (       nn.Module							):





     def __init__(				self	:      Any			,   lowercase	:      int			,   lowercase	:      int			,   lowercase	:      int			,   lowercase	:      int			,   ):





        """simple docstring"""




        super().__init__()

        lowercase_    :str					    =				height
        lowercase_    :Union[str, Any]					    =				width
        lowercase_    :Any					    =				num_embed
        lowercase_    :Dict					    =				embed_dim

        lowercase_    :int					    =				nn.Embedding(self.num_embed			,   lowercase		)
        lowercase_    :Any					    =				nn.Embedding(self.height			,   lowercase		)
        lowercase_    :List[str]					    =				nn.Embedding(self.width			,   lowercase		)







     def 							lowercase__   (				self	:      Tuple			,   lowercase	:      List[Any]		):





        """simple docstring"""




        lowercase_    :Any					    =				self.emb(lowercase		)

        lowercase_    :Dict					    =				self.height_emb(torch.arange(self.height			,   device=index.device		).view(1			,   self.height		)		)

        # 1 x H x D -> 1 x H x 1 x D
        lowercase_    :Any					    =				height_emb.unsqueeze(2		)

        lowercase_    :Union[str, Any]					    =				self.width_emb(torch.arange(self.width			,   device=index.device		).view(1			,   self.width		)		)

        # 1 x W x D -> 1 x 1 x W x D
        lowercase_    :List[str]					    =				width_emb.unsqueeze(1		)

        lowercase_    :Union[str, Any]					    =				height_emb + width_emb

        # 1 x H x W x D -> 1 x L xD
        lowercase_    :Optional[Any]					    =				pos_emb.view(1			,   self.height * self.width			,   -1		)

        lowercase_    :str					    =				emb + pos_emb[:, : emb.shape[1], :]

        return emb







class    a_     (       nn.Module							):





     def __init__(				self	:      int			,   lowercase	:      Union[str, Any]			,   lowercase	:      Dict			,   lowercase	:      Optional[int]		):





        """simple docstring"""




        super().__init__()
        lowercase_    :Union[str, Any]					    =				dropout_prob > 0
        lowercase_    :List[str]					    =				nn.Embedding(num_classes + use_cfg_embedding			,   lowercase		)
        lowercase_    :int					    =				num_classes
        lowercase_    :Optional[Any]					    =				dropout_prob





     def 							lowercase__   (				self	:      Optional[Any]			,   lowercase	:      str			,   lowercase	:      Dict=None		):





        """simple docstring"""




        if force_drop_ids is None:
           lowercase_    :Union[str, Any]					    =				torch.rand(labels.shape[0]			,   device=labels.device		) < self.dropout_prob
        else:
           lowercase_    :Dict					    =				torch.tensor(force_drop_ids == 1		)
        lowercase_    :Tuple					    =				torch.where(lowercase			,   self.num_classes			,   lowercase		)
        return labels







     def 							lowercase__   (				self	:      Tuple			,   lowercase	:      torch.LongTensor			,   lowercase	:      int=None		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.dropout_prob > 0
        if (self.training and use_dropout) or (force_drop_ids is not None):
           lowercase_    :Optional[int]					    =				self.token_drop(lowercase			,   lowercase		)
        lowercase_    :str					    =				self.embedding_table(lowercase		)
        return embeddings







class    a_     (       nn.Module							):





     def __init__(				self	:      Optional[Any]			,   lowercase	:      int = 1_024			,   lowercase	:      int = 768			,   lowercase	:      int = 768			,   lowercase	:      int = 10			,   ):





        """simple docstring"""




        super().__init__()

        lowercase_    :Any					    =				num_image_text_embeds
        lowercase_    :List[Any]					    =				nn.Linear(lowercase			,   self.num_image_text_embeds * cross_attention_dim		)
        lowercase_    :int					    =				nn.Linear(lowercase			,   lowercase		)







     def 							lowercase__   (				self	:      Optional[int]			,   lowercase	:      torch.FloatTensor			,   lowercase	:      torch.FloatTensor		):





        """simple docstring"""




        lowercase_    :List[str]					    =				text_embeds.shape[0]

        # image
        lowercase_    :List[Any]					    =				self.image_embeds(lowercase		)
        lowercase_    :List[Any]					    =				image_text_embeds.reshape(lowercase			,   self.num_image_text_embeds			,   -1		)

        # text
        lowercase_    :Tuple					    =				self.text_proj(lowercase		)

        return torch.cat([image_text_embeds, text_embeds]			,   dim=1		)







class    a_     (       nn.Module							):





     def __init__(				self	:      List[str]			,   lowercase	:      int = 768			,   lowercase	:      int = 768			,   lowercase	:      int = 32			,   ):





        """simple docstring"""




        super().__init__()

        lowercase_    :Optional[Any]					    =				num_image_text_embeds
        lowercase_    :Optional[Any]					    =				nn.Linear(lowercase			,   self.num_image_text_embeds * cross_attention_dim		)
        lowercase_    :int					    =				nn.LayerNorm(lowercase		)







     def 							lowercase__   (				self	:      List[Any]			,   lowercase	:      torch.FloatTensor		):





        """simple docstring"""




        lowercase_    :Union[str, Any]					    =				image_embeds.shape[0]

        # image
        lowercase_    :List[str]					    =				self.image_embeds(lowercase		)
        lowercase_    :Optional[int]					    =				image_embeds.reshape(lowercase			,   self.num_image_text_embeds			,   -1		)
        lowercase_    :str					    =				self.norm(lowercase		)
        return image_embeds







class    a_     (       nn.Module							):





     def __init__(				self	:      Any			,   lowercase	:      Optional[Any]			,   lowercase	:      str			,   lowercase	:      Dict=0.1		):





        """simple docstring"""




        super().__init__()

        lowercase_    :List[Any]					    =				Timesteps(num_channels=256			,   flip_sin_to_cos=lowercase			,   downscale_freq_shift=1		)
        lowercase_    :Any					    =				TimestepEmbedding(in_channels=256			,   time_embed_dim=lowercase		)
        lowercase_    :List[str]					    =				LabelEmbedding(lowercase			,   lowercase			,   lowercase		)







     def 							lowercase__   (				self	:      Tuple			,   lowercase	:      str			,   lowercase	:      Any			,   lowercase	:      Tuple=None		):





        """simple docstring"""




        lowercase_    :Union[str, Any]					    =				self.time_proj(lowercase		)
        lowercase_    :Optional[Any]					    =				self.timestep_embedder(timesteps_proj.to(dtype=lowercase		)		)  # (N, D)

        lowercase_    :Any					    =				self.class_embedder(lowercase		)  # (N, D)

        lowercase_    :int					    =				timesteps_emb + class_labels  # (N, D)

        return conditioning







class    a_     (       nn.Module							):





     def __init__(				self	:      Optional[int]			,   lowercase	:      int			,   lowercase	:      int			,   lowercase	:      int = 64		):





        """simple docstring"""




        super().__init__()
        lowercase_    :Optional[int]					    =				nn.LayerNorm(lowercase		)
        lowercase_    :Union[str, Any]					    =				AttentionPooling(lowercase			,   lowercase		)
        lowercase_    :Union[str, Any]					    =				nn.Linear(lowercase			,   lowercase		)
        lowercase_    :Any					    =				nn.LayerNorm(lowercase		)







     def 							lowercase__   (				self	:      Dict			,   lowercase	:      str		):





        """simple docstring"""




        lowercase_    :str					    =				self.norma(lowercase		)
        lowercase_    :Tuple					    =				self.pool(lowercase		)
        lowercase_    :Union[str, Any]					    =				self.proj(lowercase		)
        lowercase_    :str					    =				self.norma(lowercase		)
        return hidden_states







class    a_     (       nn.Module							):





     def __init__(				self	:      Optional[int]			,   lowercase	:      int = 768			,   lowercase	:      int = 768			,   lowercase	:      int = 1_536		):





        """simple docstring"""




        super().__init__()
        lowercase_    :Optional[Any]					    =				nn.Linear(lowercase			,   lowercase		)
        lowercase_    :Any					    =				nn.LayerNorm(lowercase		)
        lowercase_    :Any					    =				nn.Linear(lowercase			,   lowercase		)







     def 							lowercase__   (				self	:      Optional[int]			,   lowercase	:      torch.FloatTensor			,   lowercase	:      torch.FloatTensor		):





        """simple docstring"""




        lowercase_    :Optional[int]					    =				self.text_proj(lowercase		)
        lowercase_    :Tuple					    =				self.text_norm(lowercase		)

        # image
        lowercase_    :Tuple					    =				self.image_proj(lowercase		)

        return time_image_embeds + time_text_embeds







class    a_     (       nn.Module							):





     def __init__(				self	:      List[Any]			,   lowercase	:      int = 768			,   lowercase	:      int = 1_536		):





        """simple docstring"""




        super().__init__()
        lowercase_    :Optional[Any]					    =				nn.Linear(lowercase			,   lowercase		)
        lowercase_    :Optional[int]					    =				nn.LayerNorm(lowercase		)







     def 							lowercase__   (				self	:      int			,   lowercase	:      torch.FloatTensor		):





        """simple docstring"""




        lowercase_    :int					    =				self.image_proj(lowercase		)
        lowercase_    :Optional[int]					    =				self.image_norm(lowercase		)
        return time_image_embeds







class    a_     (       nn.Module							):





     def __init__(				self	:      int			,   lowercase	:      int = 768			,   lowercase	:      int = 1_536		):





        """simple docstring"""




        super().__init__()
        lowercase_    :str					    =				nn.Linear(lowercase			,   lowercase		)
        lowercase_    :str					    =				nn.LayerNorm(lowercase		)
        lowercase_    :int					    =				nn.Sequential(
            nn.Convad(3			,   16			,   3			,   padding=1		)			,   nn.SiLU()			,   nn.Convad(16			,   16			,   3			,   padding=1		)			,   nn.SiLU()			,   nn.Convad(16			,   32			,   3			,   padding=1			,   stride=2		)			,   nn.SiLU()			,   nn.Convad(32			,   32			,   3			,   padding=1		)			,   nn.SiLU()			,   nn.Convad(32			,   96			,   3			,   padding=1			,   stride=2		)			,   nn.SiLU()			,   nn.Convad(96			,   96			,   3			,   padding=1		)			,   nn.SiLU()			,   nn.Convad(96			,   256			,   3			,   padding=1			,   stride=2		)			,   nn.SiLU()			,   nn.Convad(256			,   4			,   3			,   padding=1		)			,   )







     def 							lowercase__   (				self	:      Optional[int]			,   lowercase	:      torch.FloatTensor			,   lowercase	:      torch.FloatTensor		):





        """simple docstring"""




        lowercase_    :Dict					    =				self.image_proj(lowercase		)
        lowercase_    :int					    =				self.image_norm(lowercase		)
        lowercase_    :Optional[int]					    =				self.input_hint_block(lowercase		)
        return time_image_embeds, hint







class    a_     (       nn.Module							):





     def __init__(				self	:      List[Any]			,   lowercase	:      Tuple			,   lowercase	:      int			,   lowercase	:      Tuple=None		):





        """simple docstring"""




        super().__init__()
        lowercase_    :str					    =				dtype
        lowercase_    :int					    =				nn.Parameter(torch.randn(1			,   lowercase		) / embed_dim**0.5		)
        lowercase_    :Tuple					    =				nn.Linear(lowercase			,   lowercase			,   dtype=self.dtype		)
        lowercase_    :Dict					    =				nn.Linear(lowercase			,   lowercase			,   dtype=self.dtype		)
        lowercase_    :Dict					    =				nn.Linear(lowercase			,   lowercase			,   dtype=self.dtype		)
        lowercase_    :Optional[Any]					    =				num_heads
        lowercase_    :str					    =				embed_dim // self.num_heads







     def 							lowercase__   (				self	:      Optional[int]			,   lowercase	:      Optional[int]		):





        """simple docstring"""




        lowercase_    ,							lowercase_    ,							lowercase_    :Union[str, Any]					    =				x.size()

        def shape(lowercase	:      Optional[int]		):
           # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
           lowercase_    :List[Any]					    =				x.view(lowercase			,   -1			,   self.num_heads			,   self.dim_per_head		)
           # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
           lowercase_    :Dict					    =				x.transpose(1			,   2		)
           # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
           lowercase_    :Union[str, Any]					    =				x.reshape(bs * self.num_heads			,   -1			,   self.dim_per_head		)
           # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
           lowercase_    :Dict					    =				x.transpose(1			,   2		)
           return x

        lowercase_    :Any					    =				x.mean(dim=1			,   keepdim=lowercase		) + self.positional_embedding.to(x.dtype		)
        lowercase_    :Optional[int]					    =				torch.cat([class_token, x]			,   dim=1		)  # (bs, length+1, width)

        # (bs*n_heads, class_token_length, dim_per_head)
        lowercase_    :Union[str, Any]					    =				shape(self.q_proj(lowercase		)		)
        # (bs*n_heads, length+class_token_length, dim_per_head)
        lowercase_    :Optional[int]					    =				shape(self.k_proj(lowercase		)		)
        lowercase_    :str					    =				shape(self.v_proj(lowercase		)		)

        # (bs*n_heads, class_token_length, length+class_token_length):
        lowercase_    :Optional[int]					    =				1 / math.sqrt(math.sqrt(self.dim_per_head		)		)
        lowercase_    :Tuple					    =				torch.einsum("bct,bcs->bts"			,   q * scale			,   k * scale		)  # More stable with f16 than dividing afterwards
        lowercase_    :Optional[int]					    =				torch.softmax(weight.float()			,   dim=-1		).type(weight.dtype		)

        # (bs*n_heads, dim_per_head, class_token_length)
        lowercase_    :Tuple					    =				torch.einsum("bts,bcs->bct"			,   lowercase			,   lowercase		)

        # (bs, length+1, width)
        lowercase_    :Tuple					    =				a.reshape(lowercase			,   -1			,   1		).transpose(1			,   2		)

        return a[:, 0, :]  # cls_token









'''simple docstring'''


from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


lowerCAmelCase      :  str      							=logging.get_logger(__name__)

lowerCAmelCase      :  Union[str, Any]      							={
    '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json''',
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}
class    a_     (       _lowerCAmelCase							):
     __A  				=    "blenderbot"
     __A  				=    ["past_key_values"]
     __A  				=    {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}





     def __init__(				self	:      Dict			,   lowercase	:      List[Any]=8_008			,   lowercase	:      List[str]=128			,   lowercase	:      Optional[int]=2			,   lowercase	:      Optional[int]=10_240			,   lowercase	:      Tuple=32			,   lowercase	:      int=24			,   lowercase	:      Dict=10_240			,   lowercase	:      Any=32			,   lowercase	:      Optional[int]=0.0			,   lowercase	:      Dict=0.0			,   lowercase	:      int=True			,   lowercase	:      Tuple=True			,   lowercase	:      List[str]="gelu"			,   lowercase	:      int=2_560			,   lowercase	:      Optional[Any]=0.1			,   lowercase	:      Any=0.0			,   lowercase	:      List[str]=0.0			,   lowercase	:      Optional[Any]=0.02			,   lowercase	:      Tuple=1			,   lowercase	:      Union[str, Any]=False			,   lowercase	:      str=0			,   lowercase	:      List[str]=1			,   lowercase	:      Optional[Any]=2			,   lowercase	:      Union[str, Any]=3			,   lowercase	:      List[str]=2			,   **lowercase	:      Optional[Any]			,   ):





        """simple docstring"""




        lowercase_    :str					    =				vocab_size
        lowercase_    :Any					    =				max_position_embeddings
        lowercase_    :List[Any]					    =				d_model
        lowercase_    :Union[str, Any]					    =				encoder_ffn_dim
        lowercase_    :str					    =				encoder_layers
        lowercase_    :Optional[int]					    =				encoder_attention_heads
        lowercase_    :Dict					    =				decoder_ffn_dim
        lowercase_    :str					    =				decoder_layers
        lowercase_    :int					    =				decoder_attention_heads
        lowercase_    :Union[str, Any]					    =				dropout
        lowercase_    :int					    =				attention_dropout
        lowercase_    :int					    =				activation_dropout
        lowercase_    :str					    =				activation_function
        lowercase_    :Tuple					    =				init_std
        lowercase_    :Any					    =				encoder_layerdrop
        lowercase_    :Dict					    =				decoder_layerdrop
        lowercase_    :int					    =				use_cache
        lowercase_    :Dict					    =				encoder_layers
        lowercase_    :Dict					    =				scale_embedding  # scale factor will be sqrt(d_model) if True

        super().__init__(
            pad_token_id=lowercase			,   bos_token_id=lowercase			,   eos_token_id=lowercase			,   is_encoder_decoder=lowercase			,   decoder_start_token_id=lowercase			,   encoder_no_repeat_ngram_size=lowercase			,   forced_eos_token_id=lowercase			,   **lowercase			,   )







class    a_     (       _lowerCAmelCase							):





     @property
     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        if self.task in ["default", "seq2seq-lm"]:
           lowercase_    :Optional[int]					    =				OrderedDict(
               [
                   ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                   ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
               ]		)
           if self.use_past:
              lowercase_    :int					    =				{0: "batch"}
              lowercase_    :List[str]					    =				{0: "batch", 1: "past_decoder_sequence + sequence"}
           else:
              lowercase_    :Union[str, Any]					    =				{0: "batch", 1: "decoder_sequence"}
              lowercase_    :List[Any]					    =				{0: "batch", 1: "decoder_sequence"}
           if self.use_past:
              self.fill_with_past_key_values_(lowercase			,   direction="inputs"		)
        elif self.task == "causal-lm":
           lowercase_    :List[Any]					    =				OrderedDict(
               [
                   ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                   ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
               ]		)
           if self.use_past:
              lowercase_    ,							lowercase_    :Optional[int]					    =				self.num_layers
              for i in range(lowercase		):
                 lowercase_    :List[str]					    =				{0: "batch", 2: "past_sequence + sequence"}
                 lowercase_    :Any					    =				{0: "batch", 2: "past_sequence + sequence"}
        else:
           lowercase_    :str					    =				OrderedDict(
               [
                   ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                   ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                   ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
                   ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
               ]		)

        return common_inputs





     @property
     # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
     def 							lowercase__   (				self	:      Dict		):





        """simple docstring"""




        if self.task in ["default", "seq2seq-lm"]:
           lowercase_    :Union[str, Any]					    =				super().outputs
        else:
           lowercase_    :Optional[Any]					    =				super(lowercase			,   self		).outputs
           if self.use_past:
              lowercase_    ,							lowercase_    :Any					    =				self.num_layers
              for i in range(lowercase		):
                 lowercase_    :str					    =				{0: "batch", 2: "past_sequence + sequence"}
                 lowercase_    :Any					    =				{0: "batch", 2: "past_sequence + sequence"}
        return common_outputs





     def 							lowercase__   (				self	:      Tuple			,   lowercase	:      PreTrainedTokenizer			,   lowercase	:      int = -1			,   lowercase	:      int = -1			,   lowercase	:      bool = False			,   lowercase	:      Optional[TensorType] = None			,   ):





        """simple docstring"""




        lowercase_    :int					    =				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            lowercase			,   lowercase			,   lowercase			,   lowercase			,   lowercase		)
        # Generate decoder inputs
        lowercase_    :List[Any]					    =				seq_length if not self.use_past else 1
        lowercase_    :Optional[Any]					    =				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            lowercase			,   lowercase			,   lowercase			,   lowercase			,   lowercase		)
        lowercase_    :Tuple					    =				{F'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
        lowercase_    :Any					    =				dict(**lowercase			,   **lowercase		)

        if self.use_past:
           if not is_torch_available():
              raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."		)
           else:
              import torch
           lowercase_    ,							lowercase_    :int					    =				common_inputs["input_ids"].shape
           lowercase_    :Tuple					    =				common_inputs["decoder_input_ids"].shape[1]
           lowercase_    ,							lowercase_    :Tuple					    =				self.num_attention_heads
           lowercase_    :List[Any]					    =				(
               batch,
               num_encoder_attention_heads,
               encoder_seq_length,
               self._config.hidden_size // num_encoder_attention_heads,
           )
           lowercase_    :List[Any]					    =				decoder_seq_length
           lowercase_    :str					    =				(
               batch,
               num_decoder_attention_heads,
               decoder_past_length,
               self._config.hidden_size // num_decoder_attention_heads,
           )
           lowercase_    :Optional[int]					    =				torch.cat(
               [common_inputs["decoder_attention_mask"], torch.ones(lowercase			,   lowercase		)]			,   dim=1		)
           lowercase_    :int					    =				[]
           lowercase_    ,							lowercase_    :Tuple					    =				self.num_layers

           for _ in range(lowercase		):
              common_inputs["past_key_values"].append(
                  (
                      torch.zeros(lowercase		),
                      torch.zeros(lowercase		),
                      torch.zeros(lowercase		),
                      torch.zeros(lowercase		),
                  )		)
        return common_inputs





     def 							lowercase__   (				self	:      Optional[Any]			,   lowercase	:      PreTrainedTokenizer			,   lowercase	:      int = -1			,   lowercase	:      int = -1			,   lowercase	:      bool = False			,   lowercase	:      Optional[TensorType] = None			,   ):





        """simple docstring"""




        lowercase_    :str					    =				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            lowercase			,   lowercase			,   lowercase			,   lowercase			,   lowercase		)

        if self.use_past:
           if not is_torch_available():
              raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."		)
           else:
              import torch
           lowercase_    ,							lowercase_    :List[str]					    =				common_inputs["input_ids"].shape
           lowercase_    :int					    =				seqlen
           lowercase_    ,							lowercase_    :int					    =				self.num_layers
           lowercase_    ,							lowercase_    :Union[str, Any]					    =				self.num_attention_heads
           lowercase_    :Any					    =				(
               batch,
               num_encoder_attention_heads,
               past_key_values_length,
               self._config.hidden_size // num_encoder_attention_heads,
           )
           lowercase_    :Optional[Any]					    =				common_inputs["attention_mask"].dtype
           lowercase_    :List[Any]					    =				torch.cat(
               [common_inputs["attention_mask"], torch.ones(lowercase			,   lowercase			,   dtype=lowercase		)]			,   dim=1		)
           lowercase_    :Any					    =				[
               (torch.zeros(lowercase		), torch.zeros(lowercase		)) for _ in range(lowercase		)
           ]
        return common_inputs





     def 							lowercase__   (				self	:      Dict			,   lowercase	:      PreTrainedTokenizer			,   lowercase	:      int = -1			,   lowercase	:      int = -1			,   lowercase	:      bool = False			,   lowercase	:      Optional[TensorType] = None			,   ):





        """simple docstring"""




        lowercase_    :Optional[int]					    =				compute_effective_axis_dimension(
            lowercase			,   fixed_dimension=OnnxConfig.default_fixed_batch			,   num_token_to_add=0		)

        # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
        lowercase_    :Optional[Any]					    =				tokenizer.num_special_tokens_to_add(lowercase		)
        lowercase_    :int					    =				compute_effective_axis_dimension(
            lowercase			,   fixed_dimension=OnnxConfig.default_fixed_sequence			,   num_token_to_add=lowercase		)

        # Generate dummy inputs according to compute batch and sequence
        lowercase_    :int					    =				[" ".join([tokenizer.unk_token]		) * seq_length] * batch_size
        lowercase_    :Dict					    =				dict(tokenizer(lowercase			,   return_tensors=lowercase		)		)
        return common_inputs





     def 							lowercase__   (				self	:      Optional[Any]			,   lowercase	:      PreTrainedTokenizer			,   lowercase	:      int = -1			,   lowercase	:      int = -1			,   lowercase	:      bool = False			,   lowercase	:      Optional[TensorType] = None			,   ):





        """simple docstring"""




        if self.task in ["default", "seq2seq-lm"]:
           lowercase_    :str					    =				self._generate_dummy_inputs_for_default_and_seqaseq_lm(
               lowercase			,   batch_size=lowercase			,   seq_length=lowercase			,   is_pair=lowercase			,   framework=lowercase		)

        elif self.task == "causal-lm":
           lowercase_    :Optional[Any]					    =				self._generate_dummy_inputs_for_causal_lm(
               lowercase			,   batch_size=lowercase			,   seq_length=lowercase			,   is_pair=lowercase			,   framework=lowercase		)
        else:
           lowercase_    :List[str]					    =				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
               lowercase			,   batch_size=lowercase			,   seq_length=lowercase			,   is_pair=lowercase			,   framework=lowercase		)

        return common_inputs





     def 							lowercase__   (				self	:      Optional[int]			,   lowercase	:      str			,   lowercase	:      Union[str, Any]			,   lowercase	:      int			,   lowercase	:      List[Any]		):





        """simple docstring"""




        if self.task in ["default", "seq2seq-lm"]:
           lowercase_    :Optional[int]					    =				super()._flatten_past_key_values_(lowercase			,   lowercase			,   lowercase			,   lowercase		)
        else:
           lowercase_    :Tuple					    =				super(lowercase			,   self		)._flatten_past_key_values_(
               lowercase			,   lowercase			,   lowercase			,   lowercase		)







     def 							lowercase__   (				self	:      List[Any]			,   lowercase	:      Mapping[str, Mapping[int, str]]			,   lowercase	:      str		):





        """simple docstring"""




        if direction not in ["inputs", "outputs"]:
           raise ValueError(F'direction must either be "inputs" or "outputs", but {direction} was given'		)

        lowercase_    :Any					    =				"past_key_values" if direction == "inputs" else "present"
        lowercase_    ,							lowercase_    :Optional[int]					    =				self.num_layers

        lowercase_    :str					    =				"past_encoder_sequence"
        lowercase_    :Tuple					    =				"past_decoder_sequence" if direction == "inputs" else "past_decoder_sequence + sequence"

        for i in range(lowercase		):
           lowercase_    :Dict					    =				{0: "batch", 2: decoder_sequence}
           lowercase_    :Optional[int]					    =				{0: "batch", 2: decoder_sequence}
           lowercase_    :Optional[int]					    =				{0: "batch", 2: encoder_sequence}
           lowercase_    :int					    =				{0: "batch", 2: encoder_sequence}


