


"""simple docstring"""







from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin
@dataclass
class 							lowerCamelCase       (			A_ ):
							SCREAMING_SNAKE_CASE										=      42




class 							lowerCamelCase       (			A_				,					A_ ):






							@register_to_config
							def __init__(						self :     Tuple		, lowerCamelCase_ :     int = 16		, lowerCamelCase_ :     int = 88		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     int = 1		, lowerCamelCase_ :     float = 0.0		, lowerCamelCase_ :     int = 32		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     bool = False		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     str = "geglu"		, lowerCamelCase_ :     Optional[int] = None		, lowerCamelCase_ :     bool = False		, lowerCamelCase_ :     bool = False		, lowerCamelCase_ :     bool = False		, lowerCamelCase_ :     str = "layer_norm"		, lowerCamelCase_ :     bool = True		, ):
													super().__init__()
													UpperCAmelCase__		=    use_linear_projection
													UpperCAmelCase__		=    num_attention_heads
													UpperCAmelCase__		=    attention_head_dim
													UpperCAmelCase__		=    num_attention_heads * attention_head_dim

													# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
													# Define whether input is continuous or discrete depending on configuration
													UpperCAmelCase__		=    (in_channels is not None) and (patch_size is None)
													UpperCAmelCase__		=    num_vector_embeds is not None
													UpperCAmelCase__		=    in_channels is not None and patch_size is not None

													if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
																			UpperCAmelCase__		=    (
																			    f"""The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"""
																			    ''' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'''
																			    ''' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'''
																			    ''' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'''
																			    ''' would be very nice if you could open a Pull request for the `transformer/config.json` file'''
																			)
																			deprecate('''norm_type!=num_embeds_ada_norm'''		, '''1.0.0'''		, lowerCamelCase_		, standard_warn=lowerCamelCase_				)
																			UpperCAmelCase__		=    '''ada_norm'''

													if self.is_input_continuous and self.is_input_vectorized:
																			raise ValueError(
																			    f"""Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"""
																			    ''' sure that either `in_channels` or `num_vector_embeds` is None.'''				)
													elif self.is_input_vectorized and self.is_input_patches:
																			raise ValueError(
																			    f"""Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"""
																			    ''' sure that either `num_vector_embeds` or `num_patches` is None.'''				)
													elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
																			raise ValueError(
																			    f"""Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"""
																			    f""" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."""				)

													# 2. Define input layers
													if self.is_input_continuous:
																			UpperCAmelCase__		=    in_channels

																			UpperCAmelCase__		=    torch.nn.GroupNorm(num_groups=lowerCamelCase_		, num_channels=lowerCamelCase_		, eps=1E-6		, affine=lowerCamelCase_				)
																			if use_linear_projection:
																									UpperCAmelCase__		=    nn.Linear(lowerCamelCase_		, lowerCamelCase_				)
																			else:
																									UpperCAmelCase__		=    nn.Convad(lowerCamelCase_		, lowerCamelCase_		, kernel_size=1		, stride=1		, padding=0				)
													elif self.is_input_vectorized:
																			assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
																			assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

																			UpperCAmelCase__		=    sample_size
																			UpperCAmelCase__		=    sample_size
																			UpperCAmelCase__		=    num_vector_embeds
																			UpperCAmelCase__		=    self.height * self.width

																			UpperCAmelCase__		=    ImagePositionalEmbeddings(
																			    num_embed=lowerCamelCase_		, embed_dim=lowerCamelCase_		, height=self.height		, width=self.width				)
													elif self.is_input_patches:
																			assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

																			UpperCAmelCase__		=    sample_size
																			UpperCAmelCase__		=    sample_size

																			UpperCAmelCase__		=    patch_size
																			UpperCAmelCase__		=    PatchEmbed(
																			    height=lowerCamelCase_		, width=lowerCamelCase_		, patch_size=lowerCamelCase_		, in_channels=lowerCamelCase_		, embed_dim=lowerCamelCase_		, )

													# 3. Define transformers blocks
													UpperCAmelCase__		=    nn.ModuleList(
													    [
													        BasicTransformerBlock(
													            lowerCamelCase_		, lowerCamelCase_		, lowerCamelCase_		, dropout=lowerCamelCase_		, cross_attention_dim=lowerCamelCase_		, activation_fn=lowerCamelCase_		, num_embeds_ada_norm=lowerCamelCase_		, attention_bias=lowerCamelCase_		, only_cross_attention=lowerCamelCase_		, upcast_attention=lowerCamelCase_		, norm_type=lowerCamelCase_		, norm_elementwise_affine=lowerCamelCase_		, )
													        for d in range(lowerCamelCase_				)
													    ]				)

													# 4. Define output layers
													UpperCAmelCase__		=    in_channels if out_channels is None else out_channels
													if self.is_input_continuous:
																			# TODO: should use out_channels for continuous projections
																			if use_linear_projection:
																									UpperCAmelCase__		=    nn.Linear(lowerCamelCase_		, lowerCamelCase_				)
																			else:
																									UpperCAmelCase__		=    nn.Convad(lowerCamelCase_		, lowerCamelCase_		, kernel_size=1		, stride=1		, padding=0				)
													elif self.is_input_vectorized:
																			UpperCAmelCase__		=    nn.LayerNorm(lowerCamelCase_				)
																			UpperCAmelCase__		=    nn.Linear(lowerCamelCase_		, self.num_vector_embeds - 1				)
													elif self.is_input_patches:
																			UpperCAmelCase__		=    nn.LayerNorm(lowerCamelCase_		, elementwise_affine=lowerCamelCase_		, eps=1E-6				)
																			UpperCAmelCase__		=    nn.Linear(lowerCamelCase_		, 2 * inner_dim				)
																			UpperCAmelCase__		=    nn.Linear(lowerCamelCase_		, patch_size * patch_size * self.out_channels				)






							def 					lowerCAmelCase     (						self :     List[Any]		, lowerCamelCase_ :     torch.Tensor		, lowerCamelCase_ :     Optional[torch.Tensor] = None		, lowerCamelCase_ :     Optional[torch.LongTensor] = None		, lowerCamelCase_ :     Optional[torch.LongTensor] = None		, lowerCamelCase_ :     Dict[str, Any] = None		, lowerCamelCase_ :     Optional[torch.Tensor] = None		, lowerCamelCase_ :     Optional[torch.Tensor] = None		, lowerCamelCase_ :     bool = True		, ):



													if attention_mask is not None and attention_mask.ndim == 2:
																			# assume that mask is expressed as:
																			#   (1 = keep,      0 = discard)
																			# convert mask into a bias that can be added to attention scores:
																			#       (keep = +0,     discard = -10000.0)
																			UpperCAmelCase__		=    (1 - attention_mask.to(hidden_states.dtype				)) * -10000.0
																			UpperCAmelCase__		=    attention_mask.unsqueeze(1				)

													# convert encoder_attention_mask to a bias the same way we do for attention_mask
													if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
																			UpperCAmelCase__		=    (1 - encoder_attention_mask.to(hidden_states.dtype				)) * -10000.0
																			UpperCAmelCase__		=    encoder_attention_mask.unsqueeze(1				)

													# 1. Input
													if self.is_input_continuous:
																			UpperCAmelCase__		=    hidden_states.shape
																			UpperCAmelCase__		=    hidden_states

																			UpperCAmelCase__		=    self.norm(lowerCamelCase_				)
																			if not self.use_linear_projection:
																									UpperCAmelCase__		=    self.proj_in(lowerCamelCase_				)
																									UpperCAmelCase__		=    hidden_states.shape[1]
																									UpperCAmelCase__		=    hidden_states.permute(0		, 2		, 3		, 1				).reshape(lowerCamelCase_		, height * width		, lowerCamelCase_				)
																			else:
																									UpperCAmelCase__		=    hidden_states.shape[1]
																									UpperCAmelCase__		=    hidden_states.permute(0		, 2		, 3		, 1				).reshape(lowerCamelCase_		, height * width		, lowerCamelCase_				)
																									UpperCAmelCase__		=    self.proj_in(lowerCamelCase_				)
													elif self.is_input_vectorized:
																			UpperCAmelCase__		=    self.latent_image_embedding(lowerCamelCase_				)
													elif self.is_input_patches:
																			UpperCAmelCase__		=    self.pos_embed(lowerCamelCase_				)

													# 2. Blocks
													for block in self.transformer_blocks:
																			UpperCAmelCase__		=    block(
																			    lowerCamelCase_		, attention_mask=lowerCamelCase_		, encoder_hidden_states=lowerCamelCase_		, encoder_attention_mask=lowerCamelCase_		, timestep=lowerCamelCase_		, cross_attention_kwargs=lowerCamelCase_		, class_labels=lowerCamelCase_		, )

													# 3. Output
													if self.is_input_continuous:
																			if not self.use_linear_projection:
																									UpperCAmelCase__		=    hidden_states.reshape(lowerCamelCase_		, lowerCamelCase_		, lowerCamelCase_		, lowerCamelCase_				).permute(0		, 3		, 1		, 2				).contiguous()
																									UpperCAmelCase__		=    self.proj_out(lowerCamelCase_				)
																			else:
																									UpperCAmelCase__		=    self.proj_out(lowerCamelCase_				)
																									UpperCAmelCase__		=    hidden_states.reshape(lowerCamelCase_		, lowerCamelCase_		, lowerCamelCase_		, lowerCamelCase_				).permute(0		, 3		, 1		, 2				).contiguous()

																			UpperCAmelCase__		=    hidden_states + residual
													elif self.is_input_vectorized:
																			UpperCAmelCase__		=    self.norm_out(lowerCamelCase_				)
																			UpperCAmelCase__		=    self.out(lowerCamelCase_				)
																			# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
																			UpperCAmelCase__		=    logits.permute(0		, 2		, 1				)

																			# log(p(x_0))
																			UpperCAmelCase__		=    F.log_softmax(logits.double()		, dim=1				).float()
													elif self.is_input_patches:
																			# TODO: cleanup!
																			UpperCAmelCase__		=    self.transformer_blocks[0].norma.emb(
																			    lowerCamelCase_		, lowerCamelCase_		, hidden_dtype=hidden_states.dtype				)
																			UpperCAmelCase__		=    self.proj_out_a(F.silu(lowerCamelCase_				)				).chunk(2		, dim=1				)
																			UpperCAmelCase__		=    self.norm_out(lowerCamelCase_				) * (1 + scale[:, None]) + shift[:, None]
																			UpperCAmelCase__		=    self.proj_out_a(lowerCamelCase_				)

																			# unpatchify
																			UpperCAmelCase__		=    int(hidden_states.shape[1] ** 0.5				)
																			UpperCAmelCase__		=    hidden_states.reshape(
																			    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)				)
																			UpperCAmelCase__		=    torch.einsum('''nhwpqc->nchpwq'''		, lowerCamelCase_				)
																			UpperCAmelCase__		=    hidden_states.reshape(
																			    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)				)

													if not return_dict:
																			return (output,)

													return TransformeraDModelOutput(sample=lowerCamelCase_				)



"""simple docstring"""







import safs
class 							lowerCamelCase       (			safs.SaFileSystem ):


							"""simple docstring"""