






'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_				      =							{"configuration_t5": ["T5_PRETRAINED_CONFIG_ARCHIVE_MAP", "T5Config", "T5OnnxConfig"]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE_				      =							["T5Tokenizer"]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE_				      =							["T5TokenizerFast"]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE_				      =							[
      "T5_PRETRAINED_MODEL_ARCHIVE_LIST",
      "T5EncoderModel",
      "T5ForConditionalGeneration",
      "T5Model",
      "T5PreTrainedModel",
      "load_tf_weights_in_t5",
      "T5ForQuestionAnswering",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE_				      =							[
      "TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFT5EncoderModel",
      "TFT5ForConditionalGeneration",
      "TFT5Model",
      "TFT5PreTrainedModel",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE_				      =							[
      "FlaxT5EncoderModel",
      "FlaxT5ForConditionalGeneration",
      "FlaxT5Model",
      "FlaxT5PreTrainedModel",
  ]


if TYPE_CHECKING:
  from .configuration_ta import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, TaConfig, TaOnnxConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_ta import TaTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_ta_fast import TaTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_ta import (
        T5_PRETRAINED_MODEL_ARCHIVE_LIST,
        TaEncoderModel,
        TaForConditionalGeneration,
        TaForQuestionAnswering,
        TaModel,
        TaPreTrainedModel,
        load_tf_weights_in_ta,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_ta import (
        TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFTaEncoderModel,
        TFTaForConditionalGeneration,
        TFTaModel,
        TFTaPreTrainedModel,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_ta import (
        FlaxTaEncoderModel,
        FlaxTaForConditionalGeneration,
        FlaxTaModel,
        FlaxTaPreTrainedModel,
    )


else:
  import sys

  SCREAMING_SNAKE_CASE_				      =							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)









'''simple docstring'''


import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__		):
 __a    :	List[str]									=						{}
 for name in unet.attn_processors.keys():
  __a    :	int									=						None if name.endswith('attn1.processor'		) else unet.config.cross_attention_dim
  if name.startswith('mid_block'		):
   __a    :	Tuple									=						unet.config.block_out_channels[-1]
  elif name.startswith('up_blocks'		):
   __a    :	int									=						int(name[len('up_blocks.'		)]		)
   __a    :	List[str]									=						list(reversed(unet.config.block_out_channels		)		)[block_id]
  elif name.startswith('down_blocks'		):
   __a    :	int									=						int(name[len('down_blocks.'		)]		)
   __a    :	Optional[Any]									=						unet.config.block_out_channels[block_id]
  __a    :	Optional[int]									=						(
      LoRAAttnProcessor2_0 if hasattr(SCREAMING_SNAKE_CASE__       ,		'scaled_dot_product_attention'		) else LoRAAttnProcessor
  )
  __a    :	Optional[Any]									=						lora_attn_processor_class(
      hidden_size=SCREAMING_SNAKE_CASE__       ,		cross_attention_dim=SCREAMING_SNAKE_CASE__		)
 __a    :	Tuple									=						AttnProcsLayers(SCREAMING_SNAKE_CASE__		)
 return lora_attn_procs, unet_lora_layers






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__		):
 __a    :	Dict									=						{}
 __a    :	int									=						(
     LoRAAttnProcessor2_0 if hasattr(SCREAMING_SNAKE_CASE__       ,		'scaled_dot_product_attention'		) else LoRAAttnProcessor
 )
 for name, module in text_encoder.named_modules():
  if name.endswith(SCREAMING_SNAKE_CASE__		):
   __a    :	Dict									=						lora_attn_processor_class(
       hidden_size=module.out_proj.out_features       ,		cross_attention_dim=SCREAMING_SNAKE_CASE__		)
 return text_lora_attn_procs






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__		):
 __a    :	Tuple									=						create_text_encoder_lora_attn_procs(SCREAMING_SNAKE_CASE__		)
 __a    :	Dict									=						AttnProcsLayers(SCREAMING_SNAKE_CASE__		)
 return text_encoder_lora_layers






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__=False		):
 for _, attn_proc in text_lora_attn_procs.items():
  # set up.weights
  for layer_name, layer_module in attn_proc.named_modules():
   if layer_name.endswith('_lora'		):
    __a    :	Optional[int]									=						(
        torch.randn_like(layer_module.up.weight		)
        if randn_weight
        else torch.zeros_like(layer_module.up.weight		)
    )
    __a    :	int									=						torch.nn.Parameter(SCREAMING_SNAKE_CASE__		)




class 							lowerCAmelCase					(							unittest.TestCase   ):







   """simple docstring"""







   def    __magic_name__    ( self			)    ->   int:
    torch.manual_seed(0			)
    __a    :	str									=						UNetaDConditionModel(
        block_out_channels=(32, 64)   ,   layers_per_block=2   ,   sample_size=32   ,   in_channels=4   ,   out_channels=4   ,   down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')   ,   up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')   ,   cross_attention_dim=32   ,   )
    __a    :	List[Any]									=						DDIMScheduler(
        beta_start=0.00_085   ,   beta_end=0.012   ,   beta_schedule='scaled_linear'   ,   clip_sample=_A   ,   set_alpha_to_one=_A   ,   steps_offset=1   ,   )
    torch.manual_seed(0			)
    __a    :	Optional[int]									=						AutoencoderKL(
        block_out_channels=[32, 64]   ,   in_channels=3   ,   out_channels=3   ,   down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']   ,   up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']   ,   latent_channels=4   ,   )
    __a    :	Union[str, Any]									=						CLIPTextConfig(
        bos_token_id=0   ,   eos_token_id=2   ,   hidden_size=32   ,   intermediate_size=37   ,   layer_norm_eps=1E-0_5   ,   num_attention_heads=4   ,   num_hidden_layers=5   ,   pad_token_id=1   ,   vocab_size=1000   ,   )
    __a    :	List[Any]									=						CLIPTextModel(_A			)
    __a    :	Dict									=						CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'			)

    __a			,    __a    :	Optional[int]									=						create_unet_lora_layers(_A			)
    __a    :	List[Any]									=						create_text_encoder_lora_layers(_A			)

    __a    :	str									=						{
        'unet': unet,
        'scheduler': scheduler,
        'vae': vae,
        'text_encoder': text_encoder,
        'tokenizer': tokenizer,
        'safety_checker': None,
        'feature_extractor': None,
    }
    __a    :	Optional[Any]									=						{
        'unet_lora_layers': unet_lora_layers,
        'text_encoder_lora_layers': text_encoder_lora_layers,
        'unet_lora_attn_procs': unet_lora_attn_procs,
    }
    return pipeline_components, lora_components



   def    __magic_name__    ( self			)    ->   Optional[Any]:
    __a    :	int									=						1
    __a    :	Tuple									=						10
    __a    :	int									=						4
    __a    :	List[str]									=						(32, 32)

    __a    :	Tuple									=						torch.manual_seed(0			)
    __a    :	Tuple									=						floats_tensor((batch_size, num_channels) + sizes			)
    __a    :	Tuple									=						torch.randint(1   ,   _A   ,   size=(batch_size, sequence_length)   ,   generator=_A			)

    __a    :	Union[str, Any]									=						{
        'prompt': 'A painting of a squirrel eating a burger',
        'generator': generator,
        'num_inference_steps': 2,
        'guidance_scale': 6.0,
        'output_type': 'numpy',
    }

    return noise, input_ids, pipeline_inputs

    # copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb



   def    __magic_name__    ( self			)    ->   Optional[int]:
    __a    :	Optional[Any]									=						77

    __a    :	Union[str, Any]									=						torch.randint(2   ,   56   ,   size=(1, max_seq_length)   ,   generator=torch.manual_seed(0			)			)

    __a    :	Optional[int]									=						{}
    __a    :	Tuple									=						inputs
    return prepared_inputs



   def    __magic_name__    ( self   ,   _A			)    ->   Optional[int]:
    __a			,    __a    :	Dict									=						self.get_dummy_components()
    LoraLoaderMixin.save_lora_weights(
        save_directory=_A   ,   unet_lora_layers=lora_components['unet_lora_layers']   ,   text_encoder_lora_layers=lora_components['text_encoder_lora_layers']   ,   )
    self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.bin'			)			)			)



   def    __magic_name__    ( self			)    ->   str:
    __a			,    __a    :	Dict									=						self.get_dummy_components()
    __a    :	List[Any]									=						StableDiffusionPipeline(**_A			)
    __a    :	List[Any]									=						sd_pipe.to(_A			)
    sd_pipe.set_progress_bar_config(disable=_A			)

    __a			,    __a			,    __a    :	List[Any]									=						self.get_dummy_inputs()

    __a    :	Dict									=						sd_pipe(**_A			).images
    __a    :	Optional[int]									=						original_images[0, -3:, -3:, -1]

    with tempfile.TemporaryDirectory() as tmpdirname:
     LoraLoaderMixin.save_lora_weights(
         save_directory=_A   ,   unet_lora_layers=lora_components['unet_lora_layers']   ,   text_encoder_lora_layers=lora_components['text_encoder_lora_layers']   ,   )
     self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.bin'			)			)			)
     sd_pipe.load_lora_weights(_A			)

    __a    :	str									=						sd_pipe(**_A			).images
    __a    :	Optional[Any]									=						lora_images[0, -3:, -3:, -1]

    # Outputs shouldn't match.
    self.assertFalse(torch.allclose(torch.from_numpy(_A			)   ,   torch.from_numpy(_A			)			)			)



   def    __magic_name__    ( self			)    ->   Tuple:
    __a			,    __a    :	List[str]									=						self.get_dummy_components()
    __a    :	Any									=						StableDiffusionPipeline(**_A			)
    __a    :	str									=						sd_pipe.to(_A			)
    sd_pipe.set_progress_bar_config(disable=_A			)

    __a			,    __a			,    __a    :	str									=						self.get_dummy_inputs()

    __a    :	Optional[Any]									=						sd_pipe(**_A			).images
    __a    :	Tuple									=						original_images[0, -3:, -3:, -1]

    with tempfile.TemporaryDirectory() as tmpdirname:
     LoraLoaderMixin.save_lora_weights(
         save_directory=_A   ,   unet_lora_layers=lora_components['unet_lora_layers']   ,   text_encoder_lora_layers=lora_components['text_encoder_lora_layers']   ,   safe_serialization=_A   ,   )
     self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.safetensors'			)			)			)
     sd_pipe.load_lora_weights(_A			)

    __a    :	str									=						sd_pipe(**_A			).images
    __a    :	int									=						lora_images[0, -3:, -3:, -1]

    # Outputs shouldn't match.
    self.assertFalse(torch.allclose(torch.from_numpy(_A			)   ,   torch.from_numpy(_A			)			)			)



   def    __magic_name__    ( self			)    ->   Optional[Any]:
    __a			,    __a    :	Optional[int]									=						self.get_dummy_components()
    __a    :	List[Any]									=						lora_components['unet_lora_attn_procs']
    __a    :	Union[str, Any]									=						StableDiffusionPipeline(**_A			)
    __a    :	Optional[Any]									=						sd_pipe.to(_A			)
    sd_pipe.set_progress_bar_config(disable=_A			)

    __a			,    __a			,    __a    :	Union[str, Any]									=						self.get_dummy_inputs()

    __a    :	List[Any]									=						sd_pipe(**_A			).images
    __a    :	Union[str, Any]									=						original_images[0, -3:, -3:, -1]

    with tempfile.TemporaryDirectory() as tmpdirname:
     __a    :	Optional[Any]									=						sd_pipe.unet
     unet.set_attn_processor(_A			)
     unet.save_attn_procs(_A			)
     self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.bin'			)			)			)
     sd_pipe.load_lora_weights(_A			)

    __a    :	Optional[Any]									=						sd_pipe(**_A			).images
    __a    :	Any									=						lora_images[0, -3:, -3:, -1]

    # Outputs shouldn't match.
    self.assertFalse(torch.allclose(torch.from_numpy(_A			)   ,   torch.from_numpy(_A			)			)			)



   def    __magic_name__    ( self			)    ->   int:
    __a			,    __a    :	Dict									=						self.get_dummy_components()
    __a    :	Optional[int]									=						StableDiffusionPipeline(**_A			)

    __a    :	List[Any]									=						self.get_dummy_tokens()

    # inference without lora
    __a    :	List[Any]									=						pipe.text_encoder(**_A			)[0]
    assert outputs_without_lora.shape == (1, 77, 32)

    # create lora_attn_procs with zeroed out up.weights
    __a    :	int									=						create_text_encoder_lora_attn_procs(pipe.text_encoder			)
    set_lora_up_weights(_A   ,   randn_weight=_A			)

    # monkey patch
    pipe._modify_text_encoder(_A			)

    # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
    del text_attn_procs
    gc.collect()

    # inference with lora
    __a    :	Union[str, Any]									=						pipe.text_encoder(**_A			)[0]
    assert outputs_with_lora.shape == (1, 77, 32)

    assert torch.allclose(
        _A   ,   _A			), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

    # create lora_attn_procs with randn up.weights
    __a    :	Optional[int]									=						create_text_encoder_lora_attn_procs(pipe.text_encoder			)
    set_lora_up_weights(_A   ,   randn_weight=_A			)

    # monkey patch
    pipe._modify_text_encoder(_A			)

    # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
    del text_attn_procs
    gc.collect()

    # inference with lora
    __a    :	List[str]									=						pipe.text_encoder(**_A			)[0]
    assert outputs_with_lora.shape == (1, 77, 32)

    assert not torch.allclose(
        _A   ,   _A			), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"



   def    __magic_name__    ( self			)    ->   Union[str, Any]:
    __a			,    __a    :	List[str]									=						self.get_dummy_components()
    __a    :	str									=						StableDiffusionPipeline(**_A			)

    __a    :	Tuple									=						self.get_dummy_tokens()

    # inference without lora
    __a    :	Tuple									=						pipe.text_encoder(**_A			)[0]
    assert outputs_without_lora.shape == (1, 77, 32)

    # create lora_attn_procs with randn up.weights
    __a    :	Optional[int]									=						create_text_encoder_lora_attn_procs(pipe.text_encoder			)
    set_lora_up_weights(_A   ,   randn_weight=_A			)

    # monkey patch
    pipe._modify_text_encoder(_A			)

    # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
    del text_attn_procs
    gc.collect()

    # inference with lora
    __a    :	int									=						pipe.text_encoder(**_A			)[0]
    assert outputs_with_lora.shape == (1, 77, 32)

    assert not torch.allclose(
        _A   ,   _A			), "lora outputs should be different to without lora outputs"

    # remove monkey patch
    pipe._remove_text_encoder_monkey_patch()

    # inference with removed lora
    __a    :	Union[str, Any]									=						pipe.text_encoder(**_A			)[0]
    assert outputs_without_lora_removed.shape == (1, 77, 32)

    assert torch.allclose(
        _A   ,   _A			), "remove lora monkey patch should restore the original outputs"



   def    __magic_name__    ( self			)    ->   Dict:
    __a			,    __a    :	Dict									=						self.get_dummy_components()
    __a    :	Optional[Any]									=						StableDiffusionPipeline(**_A			)
    __a    :	int									=						sd_pipe.to(_A			)
    sd_pipe.set_progress_bar_config(disable=_A			)

    __a			,    __a			,    __a    :	Any									=						self.get_dummy_inputs()

    with tempfile.TemporaryDirectory() as tmpdirname:
     LoraLoaderMixin.save_lora_weights(
         save_directory=_A   ,   unet_lora_layers=lora_components['unet_lora_layers']   ,   text_encoder_lora_layers=lora_components['text_encoder_lora_layers']   ,   )
     self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.bin'			)			)			)
     sd_pipe.load_lora_weights(_A			)

    __a    :	int									=						sd_pipe(**_A			).images
    __a    :	Any									=						lora_images[0, -3:, -3:, -1]

    __a    :	Optional[int]									=						sd_pipe(**_A   ,   cross_attention_kwargs={'scale': 0.5}			).images
    __a    :	str									=						lora_images_with_scale[0, -3:, -3:, -1]

    # Outputs shouldn't match.
    self.assertFalse(
        torch.allclose(torch.from_numpy(_A			)   ,   torch.from_numpy(_A			)			)			)



   def    __magic_name__    ( self			)    ->   str:
    with tempfile.TemporaryDirectory() as tmpdirname:
     self.create_lora_weight_file(_A			)

     __a			,    __a    :	Union[str, Any]									=						self.get_dummy_components()
     __a    :	List[str]									=						StableDiffusionPipeline(**_A			)
     __a    :	List[str]									=						sd_pipe.to(_A			)
     sd_pipe.set_progress_bar_config(disable=_A			)

     # check if vanilla attention processors are used
     for _, module in sd_pipe.unet.named_modules():
      if isinstance(_A   ,   _A			):
       self.assertIsInstance(module.processor   ,   (AttnProcessor, AttnProcessor2_0)			)

            # load LoRA weight file
     sd_pipe.load_lora_weights(_A			)

     # check if lora attention processors are used
     for _, module in sd_pipe.unet.named_modules():
      if isinstance(_A   ,   _A			):
       __a    :	str									=						(
           LoRAAttnProcessor2_0 if hasattr(_A   ,   'scaled_dot_product_attention'			) else LoRAAttnProcessor
       )
       self.assertIsInstance(module.processor   ,   _A			)



   @unittest.skipIf(torch_device != 'cuda'   ,   'This test is supposed to run on GPU'			)
   def    __magic_name__    ( self			)    ->   Union[str, Any]:
    with tempfile.TemporaryDirectory() as tmpdirname:
     self.create_lora_weight_file(_A			)

     __a			,    __a    :	Optional[int]									=						self.get_dummy_components()
     __a    :	Tuple									=						StableDiffusionPipeline(**_A			)
     __a    :	Optional[int]									=						sd_pipe.to(_A			)
     sd_pipe.set_progress_bar_config(disable=_A			)

     # enable XFormers
     sd_pipe.enable_xformers_memory_efficient_attention()

     # check if xFormers attention processors are used
     for _, module in sd_pipe.unet.named_modules():
      if isinstance(_A   ,   _A			):
       self.assertIsInstance(module.processor   ,   _A			)

            # load LoRA weight file
     sd_pipe.load_lora_weights(_A			)

     # check if lora attention processors are used
     for _, module in sd_pipe.unet.named_modules():
      if isinstance(_A   ,   _A			):
       self.assertIsInstance(module.processor   ,   _A			)



   @unittest.skipIf(torch_device != 'cuda'   ,   'This test is supposed to run on GPU'			)
   def    __magic_name__    ( self			)    ->   Union[str, Any]:
    __a			,    __a    :	Union[str, Any]									=						self.get_dummy_components()
    __a    :	Tuple									=						StableDiffusionPipeline(**_A			)
    __a    :	str									=						sd_pipe.to(_A			)
    sd_pipe.set_progress_bar_config(disable=_A			)

    __a			,    __a			,    __a    :	Optional[Any]									=						self.get_dummy_inputs()

    # enable XFormers
    sd_pipe.enable_xformers_memory_efficient_attention()

    __a    :	str									=						sd_pipe(**_A			).images
    __a    :	str									=						original_images[0, -3:, -3:, -1]

    with tempfile.TemporaryDirectory() as tmpdirname:
     LoraLoaderMixin.save_lora_weights(
         save_directory=_A   ,   unet_lora_layers=lora_components['unet_lora_layers']   ,   text_encoder_lora_layers=lora_components['text_encoder_lora_layers']   ,   )
     self.assertTrue(os.path.isfile(os.path.join(_A   ,   'pytorch_lora_weights.bin'			)			)			)
     sd_pipe.load_lora_weights(_A			)

    __a    :	Union[str, Any]									=						sd_pipe(**_A			).images
    __a    :	str									=						lora_images[0, -3:, -3:, -1]

    # Outputs shouldn't match.
    self.assertFalse(torch.allclose(torch.from_numpy(_A			)   ,   torch.from_numpy(_A			)			)			)




