
"""simple docstring"""


import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
							import torch

lowerCAmelCase			=		logging.get_logger(__name__)






def      lowerCAmelCase_     (	)     ->int:
			return importlib.util.find_spec('deepspeed'    ) is not None


if is_accelerate_available() and is_deepspeed_available():
							from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
							# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
							# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
							from builtins import object as DeepSpeedConfig

class 					A_			(  A__     ):



							"""simple docstring"""






							def __init__(						self						:Any  ,							lowerCamelCase_						:Union[str, Any] ):





										"""simple docstring"""

										set_hf_deepspeed_config(self )
										dep_version_check('accelerate' )
										dep_version_check('deepspeed' )
										super().__init__(lowerCamelCase_ )






class 					A_			(  A__     ):



							"""simple docstring"""






							def __init__(						self						:List[str]  ,							lowerCamelCase_						:Optional[Any] ):





										"""simple docstring"""

										super().__init__(lowerCamelCase_ )
										lowerCamelCase__			:  Optional[int]  =None
										lowerCamelCase__			:  Any  =[]





							def    UpperCAmelCase__							(						self						:Union[str, Any] ):





										"""simple docstring"""

										if self._dtype is None:
													raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype' )
										return self._dtype





							def    UpperCAmelCase__							(						self						:Optional[Any]  ,							lowerCamelCase_						:Any ):





										"""simple docstring"""

										lowerCamelCase__			:  Any  =self.get_value(lowerCamelCase_ )
										if val is None:
													return False
										else:
													return val == "auto"





							def    UpperCAmelCase__							(						self						:Any  ,							lowerCamelCase_						:Any  ,							lowerCamelCase_						:int  ,							lowerCamelCase_						:Any=None  ,							lowerCamelCase_						:List[str]=True ):





										"""simple docstring"""

										lowerCamelCase__ ,       lowerCamelCase__			:  Dict  =self.find_config_node(lowerCamelCase_ )
										if config is None:
													return

										if config.get(lowerCamelCase_ ) == "auto":
													lowerCamelCase__			:  Optional[Any]  =hf_val
													return

										if not must_match:
													return

										lowerCamelCase__			:  str  =config.get(lowerCamelCase_ )
										if ds_val is not None and ds_val != hf_val:
													self.mismatches.append(f"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}""" )

							SCREAMING_SNAKE_CASE_			   =     partialmethod(A__   ,      must_match=A__     )





							def    UpperCAmelCase__							(						self						:Union[str, Any]  ,							lowerCamelCase_						:str ):





										"""simple docstring"""

										lowerCamelCase__			:  List[Any]  =args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
										self.fill_match(
										    'train_micro_batch_size_per_gpu'  ,							args.per_device_train_batch_size  ,							'per_device_train_batch_size' )
										self.fill_match('gradient_accumulation_steps'  ,							args.gradient_accumulation_steps  ,							'gradient_accumulation_steps' )
										self.fill_match('train_batch_size'  ,							lowerCamelCase_  ,							'train_batch_size (calculated)' )
										self.fill_match('gradient_clipping'  ,							args.max_grad_norm  ,							'max_grad_norm' )

										self.fill_match('optimizer.params.lr'  ,							args.learning_rate  ,							'learning_rate' )
										self.fill_match('optimizer.params.betas'  ,							[args.adam_betaa, args.adam_betaa]  ,							'adam_beta1+adam_beta2' )
										self.fill_match('optimizer.params.eps'  ,							args.adam_epsilon  ,							'adam_epsilon' )
										self.fill_match('optimizer.params.weight_decay'  ,							args.weight_decay  ,							'weight_decay' )

										self.fill_only('scheduler.params.warmup_min_lr'  ,							0 )  # not a trainer arg
										self.fill_match('scheduler.params.warmup_max_lr'  ,							args.learning_rate  ,							'learning_rate' )
										# total_num_steps - will get set in trainer_config_finalize

										# fp16
										if args.fpaa or args.fpaa_full_eval:
													lowerCamelCase__			:  Optional[int]  ='apex' if args.fpaa_backend == 'apex' else 'amp'
										else:
													lowerCamelCase__			:  Union[str, Any]  =None

										if args.save_on_each_node:
													# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
													lowerCamelCase__			:  int  =self.config.get('checkpoint'  ,							{} )
													lowerCamelCase__			:  List[str]  =args.save_on_each_node

										# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
										# any here unless the user did the work
										self.fill_match(
										    'fp16.enabled'  ,							((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')  ,							'fp16|fp16_full_eval+fp16_backend(amp)'  ,							)

										# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
										# ZeRO features
										self.fill_match('amp.enabled'  ,							fpaa_backend == 'apex'  ,							'fp16+fp16_backend(apex)' )
										self.fill_match('amp.opt_level'  ,							args.fpaa_opt_level  ,							'fp16_opt_level' )

										self.fill_match('bf16.enabled'  ,							(args.bfaa or args.bfaa_full_eval)  ,							'bf16|bf16_full_eval' )

										# deepspeed's default mode is fp16 unless there is a config that says differently
										if self.is_true('bf16.enabled' ):
													lowerCamelCase__			:  int  =torch.bfloataa
										elif self.is_false('fp16.enabled' ):
													lowerCamelCase__			:  Optional[int]  =torch.floataa
										else:
													lowerCamelCase__			:  Optional[Any]  =torch.floataa





							def    UpperCAmelCase__							(						self						:int  ,							lowerCamelCase_						:int  ,							lowerCamelCase_						:Union[str, Any]  ,							lowerCamelCase_						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  int  =[
										    'zero_optimization.reduce_bucket_size',
										    'zero_optimization.stage3_prefetch_bucket_size',
										    'zero_optimization.stage3_param_persistence_threshold',
										]
										lowerCamelCase__			:  Dict  =[x for x in hidden_size_based_keys if self.is_auto(lowerCamelCase_ )]

										if len(lowerCamelCase_ ) > 0:
													if hasattr(model.config  ,							'hidden_size' ):
																lowerCamelCase__			:  List[Any]  =model.config.hidden_size
													elif hasattr(model.config  ,							'hidden_sizes' ):
																# if there are many hidden sizes pick the largest one
																lowerCamelCase__			:  Tuple  =max(model.config.hidden_sizes )
													else:
																raise ValueError(
																    'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
																    'therefore it\'s not possible to automatically fill out the following `auto` entries '
																    f"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
																    '`auto` values for these keys with an integer value of your choice.' )

													self.fill_only('zero_optimization.reduce_bucket_size'  ,							hidden_size * hidden_size )
													if self.is_zeroa():
																# automatically assign the optimal config values based on model config
																self.fill_only('zero_optimization.stage3_prefetch_bucket_size'  ,							0.9 * hidden_size * hidden_size )
																self.fill_only('zero_optimization.stage3_param_persistence_threshold'  ,							10 * hidden_size )

        # scheduler
										self.fill_match('scheduler.params.total_num_steps'  ,							lowerCamelCase_  ,							'num_training_steps (calculated)' )
										self.fill_match('scheduler.params.warmup_num_steps'  ,							args.get_warmup_steps(lowerCamelCase_ )  ,							'warmup_steps' )

										if len(self.mismatches ) > 0:
													lowerCamelCase__			:  int  ='\n'.join(self.mismatches )
													raise ValueError(
													    'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
													    f""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'.""" )






lowerCAmelCase			=		None






def      lowerCAmelCase_     (	snake_case_	:    List[Any]    )     ->Any:
			# this is a special weakref global object to allow us to get to Deepspeed config from APIs
			# that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
			global _hf_deepspeed_config_weak_ref
			# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
			lowerCamelCase__			:  Union[str, Any]  =weakref.ref(snake_case_    )






def      lowerCAmelCase_     (	)     ->Optional[Any]:
			# useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
			global _hf_deepspeed_config_weak_ref
			lowerCamelCase__			:  Optional[int]  =None






def      lowerCAmelCase_     (	)     ->str:
			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().is_zeroa()
			else:
						return False






def      lowerCAmelCase_     (	)     ->str:
			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().config
			else:
						return None






def      lowerCAmelCase_     (	snake_case_	:    Tuple							,					snake_case_	:    Optional[int]							,					snake_case_	:    Optional[Any]							,					snake_case_	:    List[Any]							,					snake_case_	:    Optional[int]    )     ->Optional[int]:
			from accelerate.utils import DummyOptim, DummyScheduler

			lowerCamelCase__			:  Optional[Any]  =hf_deepspeed_config.config

			# Optimizer + Scheduler
			# Currently supported combos:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Yes
			# 3. DS scheduler + HF optimizer: Yes
			# 4. HF scheduler + DS optimizer: No
			#
			# Unless Offload is enabled in which case it's:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Mostly*
			# 3. DS scheduler + HF optimizer: Mostly*
			# 4. HF scheduler + DS optimizer: No
			#
			# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

			lowerCamelCase__			:  List[Any]  =None
			if "optimizer" in config:
						if args.adafactor:
									raise ValueError(
									    '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
									    'Only one optimizer can be configured.'    )
						lowerCamelCase__			:  Union[str, Any]  =DummyOptim(params=snake_case_    )
			else:
						if hf_deepspeed_config.is_offload():
									logger.info(
									    'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
									    ' custom optimizer has both CPU and GPU implementation (except LAMB)'    )

						# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
						# But trainer uses AdamW by default.
						lowerCamelCase__			:  Any  =trainer.create_optimizer()
						# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
						lowerCamelCase__			:  Dict  =True

			lowerCamelCase__			:  Union[str, Any]  =None
			if "scheduler" in config:
						lowerCamelCase__			:  List[str]  =DummyScheduler(snake_case_    )
			else:
						if isinstance(snake_case_							,					snake_case_    ):
									raise ValueError(
									    'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
									    'Please configure a scheduler in the DeepSpeed config.'    )
						lowerCamelCase__			:  List[str]  =trainer.create_scheduler(num_training_steps=snake_case_							,					optimizer=snake_case_    )

			return optimizer, lr_scheduler






def      lowerCAmelCase_     (	snake_case_	:    Optional[int]							,					snake_case_	:    Dict							,					snake_case_	:    Any=False    )     ->Any:
			from deepspeed.utils import logger as ds_logger

			lowerCamelCase__			:  str  =trainer.model
			lowerCamelCase__			:  Union[str, Any]  =trainer.args

			lowerCamelCase__			:  List[str]  =trainer.accelerator.state.deepspeed_plugin.hf_ds_config

			# resume config update - some bits like `model` and `num_training_steps` only become available during train
			hf_deepspeed_config.trainer_config_finalize(snake_case_							,					snake_case_							,					snake_case_    )

			# set the Deepspeed log level consistent with the Trainer
			ds_logger.setLevel(args.get_process_log_level()    )

			if inference:
						# only Z3 makes sense for the inference
						if not hf_deepspeed_config.is_zeroa():
									raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'    )

						# in case the training config is re-used for inference
						hf_deepspeed_config.del_config_sub_tree('optimizer'    )
						hf_deepspeed_config.del_config_sub_tree('lr_scheduler'    )
						lowerCamelCase__ ,       lowerCamelCase__			:  str  =None, None
						lowerCamelCase__			:  List[Any]  =None
			else:
						lowerCamelCase__			:  Optional[Any]  =None  # important for when deepspeed_init is used as re-init
						lowerCamelCase__			:  List[Any]  =list(filter(lambda snake_case_    : p.requires_grad							,					model.parameters()    )    )
						lowerCamelCase__ ,       lowerCamelCase__			:  Tuple  =deepspeed_optim_sched(
						    snake_case_							,					snake_case_							,					snake_case_							,					snake_case_							,					snake_case_    )

			# keep for quick debug:
			# from pprint import pprint; pprint(config)

			return optimizer, lr_scheduler






def      lowerCAmelCase_     (	snake_case_	:    Optional[int]							,					snake_case_	:    List[Any]    )     ->int:
			# it's possible that the user is trying to resume from model_path, which doesn't necessarily
			# contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
			# a resume from a checkpoint and not just a local pretrained weight. So we check here if the
			# path contains what looks like a deepspeed checkpoint
			import glob

			lowerCamelCase__			:  List[Any]  =sorted(glob.glob(f"""{checkpoint_path}/global_step*"""    )    )

			if len(snake_case_    ) > 0:
						logger.info(f"""Attempting to resume from {checkpoint_path}"""    )
						# this magically updates self.optimizer and self.lr_scheduler
						lowerCamelCase__ ,       lowerCamelCase__			:  Dict  =deepspeed_engine.load_checkpoint(
						    snake_case_							,					load_optimizer_states=snake_case_							,					load_lr_scheduler_states=snake_case_    )
						if load_path is None:
									raise ValueError(f"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""    )
			else:
						raise ValueError(f"""Can't find a valid checkpoint at {checkpoint_path}"""    )

"""simple docstring"""


from typing import Any, Callable, Dict, List, Optional, Union

import PIL.Image
import torch
from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionImgaImgPipeline,
    StableDiffusionInpaintPipelineLegacy,
    StableDiffusionPipeline,
    UNetaDConditionModel,
)
from diffusers.configuration_utils import FrozenDict
from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
from diffusers.utils import deprecate, logging


lowerCAmelCase			=		logging.get_logger(__name__)  # pylint: disable=invalid-name

class 					A_			(  A__     ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_			   =     ["""safety_checker""", """feature_extractor"""]





							def __init__(						self						:Optional[Any]  ,							lowerCamelCase_						:AutoencoderKL  ,							lowerCamelCase_						:CLIPTextModel  ,							lowerCamelCase_						:CLIPTokenizer  ,							lowerCamelCase_						:UNetaDConditionModel  ,							lowerCamelCase_						:Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler]  ,							lowerCamelCase_						:StableDiffusionSafetyChecker  ,							lowerCamelCase_						:CLIPImageProcessor  ,							lowerCamelCase_						:bool = True  ,							):





										"""simple docstring"""

										super().__init__()
										if hasattr(scheduler.config  ,							'steps_offset' ) and scheduler.config.steps_offset != 1:
													lowerCamelCase__			:  Dict  =(
													    f"""The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"""
													    f""" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure """
													    'to update the config accordingly as leaving `steps_offset` might led to incorrect results'
													    ' in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,'
													    ' it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`'
													    ' file'
													)
													deprecate('steps_offset!=1'  ,							'1.0.0'  ,							lowerCamelCase_  ,							standard_warn=lowerCamelCase_ )
													lowerCamelCase__			:  Tuple  =dict(scheduler.config )
													lowerCamelCase__			:  str  =1
													lowerCamelCase__			:  List[Any]  =FrozenDict(lowerCamelCase_ )

										self.register_modules(
										    vae=lowerCamelCase_  ,							text_encoder=lowerCamelCase_  ,							tokenizer=lowerCamelCase_  ,							unet=lowerCamelCase_  ,							scheduler=lowerCamelCase_  ,							safety_checker=lowerCamelCase_  ,							feature_extractor=lowerCamelCase_  ,							)
										self.register_to_config(requires_safety_checker=lowerCamelCase_ )





							@property
							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										return {k: getattr(self  ,							lowerCamelCase_ ) for k in self.config.keys() if not k.startswith('_' )}





							def    UpperCAmelCase__							(						self						:Optional[Any]  ,							lowerCamelCase_						:Optional[Union[str, int]] = "auto" ):





										"""simple docstring"""

										if slice_size == "auto":
													# half the attention head size is usually a good trade-off between
													# speed and memory
													lowerCamelCase__			:  List[Any]  =self.unet.config.attention_head_dim // 2
										self.unet.set_attention_slice(lowerCamelCase_ )





							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										self.enable_attention_slicing(lowerCamelCase_ )





							@torch.no_grad()
							def    UpperCAmelCase__							(						self						:Optional[int]  ,							lowerCamelCase_						:Union[str, List[str]]  ,							lowerCamelCase_						:Union[torch.FloatTensor, PIL.Image.Image]  ,							lowerCamelCase_						:Union[torch.FloatTensor, PIL.Image.Image]  ,							lowerCamelCase_						:float = 0.8  ,							lowerCamelCase_						:Optional[int] = 50  ,							lowerCamelCase_						:Optional[float] = 7.5  ,							lowerCamelCase_						:Optional[Union[str, List[str]]] = None  ,							lowerCamelCase_						:Optional[int] = 1  ,							lowerCamelCase_						:Optional[float] = 0.0  ,							lowerCamelCase_						:Optional[torch.Generator] = None  ,							lowerCamelCase_						:Optional[str] = "pil"  ,							lowerCamelCase_						:bool = True  ,							lowerCamelCase_						:Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,							lowerCamelCase_						:int = 1  ,							):





										"""simple docstring"""

										return StableDiffusionInpaintPipelineLegacy(**self.components )(
										    prompt=lowerCamelCase_  ,							image=lowerCamelCase_  ,							mask_image=lowerCamelCase_  ,							strength=lowerCamelCase_  ,							num_inference_steps=lowerCamelCase_  ,							guidance_scale=lowerCamelCase_  ,							negative_prompt=lowerCamelCase_  ,							num_images_per_prompt=lowerCamelCase_  ,							eta=lowerCamelCase_  ,							generator=lowerCamelCase_  ,							output_type=lowerCamelCase_  ,							return_dict=lowerCamelCase_  ,							callback=lowerCamelCase_  ,							)





							@torch.no_grad()
							def    UpperCAmelCase__							(						self						:Any  ,							lowerCamelCase_						:Union[str, List[str]]  ,							lowerCamelCase_						:Union[torch.FloatTensor, PIL.Image.Image]  ,							lowerCamelCase_						:float = 0.8  ,							lowerCamelCase_						:Optional[int] = 50  ,							lowerCamelCase_						:Optional[float] = 7.5  ,							lowerCamelCase_						:Optional[Union[str, List[str]]] = None  ,							lowerCamelCase_						:Optional[int] = 1  ,							lowerCamelCase_						:Optional[float] = 0.0  ,							lowerCamelCase_						:Optional[torch.Generator] = None  ,							lowerCamelCase_						:Optional[str] = "pil"  ,							lowerCamelCase_						:bool = True  ,							lowerCamelCase_						:Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,							lowerCamelCase_						:int = 1  ,							**lowerCamelCase_						:str  ,							):





										"""simple docstring"""

										return StableDiffusionImgaImgPipeline(**self.components )(
										    prompt=lowerCamelCase_  ,							image=lowerCamelCase_  ,							strength=lowerCamelCase_  ,							num_inference_steps=lowerCamelCase_  ,							guidance_scale=lowerCamelCase_  ,							negative_prompt=lowerCamelCase_  ,							num_images_per_prompt=lowerCamelCase_  ,							eta=lowerCamelCase_  ,							generator=lowerCamelCase_  ,							output_type=lowerCamelCase_  ,							return_dict=lowerCamelCase_  ,							callback=lowerCamelCase_  ,							callback_steps=lowerCamelCase_  ,							)





							@torch.no_grad()
							def    UpperCAmelCase__							(						self						:Optional[int]  ,							lowerCamelCase_						:Union[str, List[str]]  ,							lowerCamelCase_						:int = 512  ,							lowerCamelCase_						:int = 512  ,							lowerCamelCase_						:int = 50  ,							lowerCamelCase_						:float = 7.5  ,							lowerCamelCase_						:Optional[Union[str, List[str]]] = None  ,							lowerCamelCase_						:Optional[int] = 1  ,							lowerCamelCase_						:float = 0.0  ,							lowerCamelCase_						:Optional[torch.Generator] = None  ,							lowerCamelCase_						:Optional[torch.FloatTensor] = None  ,							lowerCamelCase_						:Optional[str] = "pil"  ,							lowerCamelCase_						:bool = True  ,							lowerCamelCase_						:Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,							lowerCamelCase_						:int = 1  ,							):





										"""simple docstring"""

										return StableDiffusionPipeline(**self.components )(
										    prompt=lowerCamelCase_  ,							height=lowerCamelCase_  ,							width=lowerCamelCase_  ,							num_inference_steps=lowerCamelCase_  ,							guidance_scale=lowerCamelCase_  ,							negative_prompt=lowerCamelCase_  ,							num_images_per_prompt=lowerCamelCase_  ,							eta=lowerCamelCase_  ,							generator=lowerCamelCase_  ,							latents=lowerCamelCase_  ,							output_type=lowerCamelCase_  ,							return_dict=lowerCamelCase_  ,							callback=lowerCamelCase_  ,							callback_steps=lowerCamelCase_  ,							)