




import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, logging, randn_tensor
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin


lowercase_			:       List[Any]						       =				logging.get_logger(__name__)  # pylint: disable=invalid-name






@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerAncestralDiscrete
class 					__lowerCAmelCase				( UpperCAmelCase__				):
			snake_case_   :				torch.FloatTensor
			snake_case_   :				Optional[torch.FloatTensor]				  =  None




def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_=0.999			,					snake_case_="cosine"			,					):

						'''simple docstring'''


						if alpha_transform_type == "cosine":

												def alpha_bar_fn(snake_case_		):
																		return math.cos((t + 0.008) / 1.008 * math.pi / 2		) ** 2

						elif alpha_transform_type == "exp":

												def alpha_bar_fn(snake_case_		):
																		return math.exp(t * -12.0		)

						else:
												raise ValueError(f"""Unsupported alpha_tranform_type: {alpha_transform_type}"""		)

						_UpperCAmelCase									=  []
						for i in range(snake_case_		):
												_UpperCAmelCase									=  i / num_diffusion_timesteps
												_UpperCAmelCase									=  (i + 1) / num_diffusion_timesteps
												betas.append(min(1 - alpha_bar_fn(snake_case_		) / alpha_bar_fn(snake_case_		)			,					snake_case_		)		)
						return torch.tensor(snake_case_			,					dtype=torch.floataa		)






class 					__lowerCAmelCase				( UpperCAmelCase__      ,	UpperCAmelCase__				):
			snake_case_   :				Dict				  =  [e.name for e in KarrasDiffusionSchedulers]
			snake_case_   :				List[str]				  =  1







			@register_to_config
			def __init__( self					:    Dict					,					snake_case__					:    int = 1_000					,					snake_case__					:    float = 0.0_001					,					snake_case__					:    float = 0.02					,					snake_case__					:    str = "linear"					,					snake_case__					:    Optional[Union[np.ndarray, List[float]]] = None					,					snake_case__					:    str = "epsilon"					,					snake_case__					:    str = "linspace"					,					snake_case__					:    int = 0					,					):




									"""simple docstring"""


									if trained_betas is not None:
															_UpperCAmelCase									=  torch.tensor(snake_case__					,					dtype=torch.floataa   )
									elif beta_schedule == "linear":
															_UpperCAmelCase									=  torch.linspace(snake_case__					,					snake_case__					,					snake_case__					,					dtype=torch.floataa   )
									elif beta_schedule == "scaled_linear":
															# this schedule is very specific to the latent diffusion model.
															_UpperCAmelCase									=  (
															    torch.linspace(beta_start**0.5					,					beta_end**0.5					,					snake_case__					,					dtype=torch.floataa   ) ** 2
															)
									elif beta_schedule == "squaredcos_cap_v2":
															# Glide cosine schedule
															_UpperCAmelCase									=  betas_for_alpha_bar(snake_case__   )
									else:
															raise NotImplementedError(F"""{beta_schedule} does is not implemented for {self.__class__}"""   )

									_UpperCAmelCase									=  1.0 - self.betas
									_UpperCAmelCase									=  torch.cumprod(self.alphas					,					dim=0   )

									_UpperCAmelCase									=  np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5   )
									_UpperCAmelCase									=  np.concatenate([sigmas[::-1], [0.0]]   ).astype(np.floataa   )
									_UpperCAmelCase									=  torch.from_numpy(snake_case__   )

									# setable values
									_UpperCAmelCase									=  None
									_UpperCAmelCase									=  np.linspace(0					,					num_train_timesteps - 1					,					snake_case__					,					dtype=snake_case__   )[::-1].copy()
									_UpperCAmelCase									=  torch.from_numpy(snake_case__   )
									_UpperCAmelCase									=  False







			@property
			def UpperCamelCase    ( self					:    Optional[int]   ):




									"""simple docstring"""


									if self.config.timestep_spacing in ["linspace", "trailing"]:
															return self.sigmas.max()

									return (self.sigmas.max() ** 2 + 1) ** 0.5







			def UpperCamelCase    ( self					:    Tuple					,					snake_case__					:    torch.FloatTensor					,					snake_case__					:    Union[float, torch.FloatTensor]   ):




									"""simple docstring"""


									if isinstance(snake_case__					,					torch.Tensor   ):
															_UpperCAmelCase									=  timestep.to(self.timesteps.device   )
									_UpperCAmelCase									=  (self.timesteps == timestep).nonzero().item()
									_UpperCAmelCase									=  self.sigmas[step_index]
									_UpperCAmelCase									=  sample / ((sigma**2 + 1) ** 0.5)
									_UpperCAmelCase									=  True
									return sample







			def UpperCamelCase    ( self					:    List[str]					,					snake_case__					:    int					,					snake_case__					:    Union[str, torch.device] = None   ):




									"""simple docstring"""


									_UpperCAmelCase									=  num_inference_steps

									# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
									if self.config.timestep_spacing == "linspace":
															_UpperCAmelCase									=  np.linspace(0					,					self.config.num_train_timesteps - 1					,					snake_case__					,					dtype=snake_case__   )[
															    ::-1
															].copy()
									elif self.config.timestep_spacing == "leading":
															_UpperCAmelCase									=  self.config.num_train_timesteps // self.num_inference_steps
															# creates integer timesteps by multiplying by ratio
															# casting to int to avoid issues when num_inference_step is power of 3
															_UpperCAmelCase									=  (np.arange(0					,					snake_case__   ) * step_ratio).round()[::-1].copy().astype(snake_case__   )
															timesteps += self.config.steps_offset
									elif self.config.timestep_spacing == "trailing":
															_UpperCAmelCase									=  self.config.num_train_timesteps / self.num_inference_steps
															# creates integer timesteps by multiplying by ratio
															# casting to int to avoid issues when num_inference_step is power of 3
															_UpperCAmelCase									=  (np.arange(self.config.num_train_timesteps					,					0					,					-step_ratio   )).round().copy().astype(snake_case__   )
															timesteps -= 1
									else:
															raise ValueError(
															    F"""{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'."""   )

									_UpperCAmelCase									=  np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5   )
									_UpperCAmelCase									=  np.interp(snake_case__					,					np.arange(0					,					len(snake_case__   )   )					,					snake_case__   )
									_UpperCAmelCase									=  np.concatenate([sigmas, [0.0]]   ).astype(np.floataa   )
									_UpperCAmelCase									=  torch.from_numpy(snake_case__   ).to(device=snake_case__   )
									if str(snake_case__   ).startswith("mps"   ):
															# mps does not support float64
															_UpperCAmelCase									=  torch.from_numpy(snake_case__   ).to(snake_case__					,					dtype=torch.floataa   )
									else:
															_UpperCAmelCase									=  torch.from_numpy(snake_case__   ).to(device=snake_case__   )







			def UpperCamelCase    ( self					:    List[str]					,					snake_case__					:    torch.FloatTensor					,					snake_case__					:    Union[float, torch.FloatTensor]					,					snake_case__					:    torch.FloatTensor					,					snake_case__					:    Optional[torch.Generator] = None					,					snake_case__					:    bool = True					,					):




									"""simple docstring"""


									if (
									    isinstance(snake_case__					,					snake_case__   )
									    or isinstance(snake_case__					,					torch.IntTensor   )
									    or isinstance(snake_case__					,					torch.LongTensor   )
									):
															raise ValueError(
															    (
															        "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
															        " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass"
															        " one of the `scheduler.timesteps` as a timestep."
															    )					,					)

									if not self.is_scale_input_called:
															logger.warning(
															    "The `scale_model_input` function should be called before `step` to ensure correct denoising. "
															    "See `StableDiffusionPipeline` for a usage example."   )

									if isinstance(snake_case__					,					torch.Tensor   ):
															_UpperCAmelCase									=  timestep.to(self.timesteps.device   )

									_UpperCAmelCase									=  (self.timesteps == timestep).nonzero().item()
									_UpperCAmelCase									=  self.sigmas[step_index]

									# 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
									if self.config.prediction_type == "epsilon":
															_UpperCAmelCase									=  sample - sigma * model_output
									elif self.config.prediction_type == "v_prediction":
															# * c_out + input * c_skip
															_UpperCAmelCase									=  model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1))
									elif self.config.prediction_type == "sample":
															raise NotImplementedError("prediction_type not implemented yet: sample"   )
									else:
															raise ValueError(
															    F"""prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`"""   )

									_UpperCAmelCase									=  self.sigmas[step_index]
									_UpperCAmelCase									=  self.sigmas[step_index + 1]
									_UpperCAmelCase									=  (sigma_to**2 * (sigma_from**2 - sigma_to**2) / sigma_from**2) ** 0.5
									_UpperCAmelCase									=  (sigma_to**2 - sigma_up**2) ** 0.5

									# 2. Convert to an ODE derivative
									_UpperCAmelCase									=  (sample - pred_original_sample) / sigma

									_UpperCAmelCase									=  sigma_down - sigma

									_UpperCAmelCase									=  sample + derivative * dt

									_UpperCAmelCase									=  model_output.device
									_UpperCAmelCase									=  randn_tensor(model_output.shape					,					dtype=model_output.dtype					,					device=snake_case__					,					generator=snake_case__   )

									_UpperCAmelCase									=  prev_sample + noise * sigma_up

									if not return_dict:
															return (prev_sample,)

									return EulerAncestralDiscreteSchedulerOutput(
									    prev_sample=snake_case__					,					pred_original_sample=snake_case__   )







			def UpperCamelCase    ( self					:    List[str]					,					snake_case__					:    torch.FloatTensor					,					snake_case__					:    torch.FloatTensor					,					snake_case__					:    torch.FloatTensor					,					):




									"""simple docstring"""


									_UpperCAmelCase									=  self.sigmas.to(device=original_samples.device					,					dtype=original_samples.dtype   )
									if original_samples.device.type == "mps" and torch.is_floating_point(snake_case__   ):
															# mps does not support float64
															_UpperCAmelCase									=  self.timesteps.to(original_samples.device					,					dtype=torch.floataa   )
															_UpperCAmelCase									=  timesteps.to(original_samples.device					,					dtype=torch.floataa   )
									else:
															_UpperCAmelCase									=  self.timesteps.to(original_samples.device   )
															_UpperCAmelCase									=  timesteps.to(original_samples.device   )

									_UpperCAmelCase									=  [(schedule_timesteps == t).nonzero().item() for t in timesteps]

									_UpperCAmelCase									=  sigmas[step_indices].flatten()
									while len(sigma.shape   ) < len(original_samples.shape   ):
															_UpperCAmelCase									=  sigma.unsqueeze(-1   )

									_UpperCAmelCase									=  original_samples + noise * sigma
									return noisy_samples







			def __len__( self					:    str   ):




									"""simple docstring"""


									return self.config.num_train_timesteps










import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
				import torch

				from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
				import pyanvml.pyanvml as nvml


lowercase_			:       int						       =				logging.get_logger(__name__)






class 					__lowerCAmelCase				( UpperCAmelCase__				):
			snake_case_   :				PyTorchBenchmarkArguments
			snake_case_   :				PretrainedConfig
			snake_case_   :				str				  =  "PyTorch"







			@property
			def UpperCamelCase    ( self					:    List[Any]   ):




									"""simple docstring"""


									return torch.__version__







			def UpperCamelCase    ( self					:    str					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self._prepare_inference_func(snake_case__					,					snake_case__					,					snake_case__   )
									return self._measure_speed(_inference   )







			def UpperCamelCase    ( self					:    List[Any]					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self._prepare_inference_func(snake_case__					,					snake_case__					,					snake_case__   )
									return self._measure_memory(_inference   )







			def UpperCamelCase    ( self					:    Any					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self._prepare_train_func(snake_case__					,					snake_case__					,					snake_case__   )
									return self._measure_speed(_train   )







			def UpperCamelCase    ( self					:    List[str]					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self._prepare_train_func(snake_case__					,					snake_case__					,					snake_case__   )
									return self._measure_memory(_train   )







			def UpperCamelCase    ( self					:    Dict					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.config_dict[model_name]

									if self.args.torchscript:
															_UpperCAmelCase									=  True

									_UpperCAmelCase									=  (
									    hasattr(snake_case__					,					"architectures"   )
									    and isinstance(config.architectures					,					snake_case__   )
									    and len(config.architectures   ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
															try:
																					_UpperCAmelCase									=  config.architectures[0]
																					_UpperCAmelCase									=  __import__("transformers"					,					fromlist=[model_class]   )
																					_UpperCAmelCase									=  getattr(snake_case__					,					snake_case__   )
																					_UpperCAmelCase									=  model_cls(snake_case__   )
															except ImportError:
																					raise ImportError(
																					    F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																					    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."   )
									else:
															_UpperCAmelCase									=  MODEL_MAPPING[config.__class__](snake_case__   )

									model.eval()
									model.to(self.args.device   )

									# encoder-decoder has vocab size saved differently
									_UpperCAmelCase									=  config.vocab_size if hasattr(snake_case__					,					"vocab_size"   ) else config.encoder.vocab_size
									_UpperCAmelCase									=  torch.randint(snake_case__					,					(batch_size, sequence_length)					,					dtype=torch.long					,					device=self.args.device   )

									if self.args.fpaa:
															logger.info("Running training in Mixed Precision..."   )
															if not self.args.is_gpu:
																					raise ValueError("Mixed precision is possible only for GPU."   )
															# amp seems to have memory leaks so that memory usage
															# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
															model.half()

									if self.args.torchscript:
															with torch.no_grad():
																					_UpperCAmelCase									=  torch.jit.trace(snake_case__					,					snake_case__   )
									else:
															_UpperCAmelCase									=  model

									def encoder_decoder_forward():
															with torch.no_grad():
																					_UpperCAmelCase									=  inference_model(snake_case__					,					decoder_input_ids=snake_case__   )
															return outputs

									def encoder_forward():
															with torch.no_grad():
																					_UpperCAmelCase									=  inference_model(snake_case__   )
															return outputs

									_UpperCAmelCase									=  encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward







			def UpperCamelCase    ( self					:    str					,					snake_case__					:    str					,					snake_case__					:    int					,					snake_case__					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.config_dict[model_name]

									_UpperCAmelCase									=  (
									    hasattr(snake_case__					,					"architectures"   )
									    and isinstance(config.architectures					,					snake_case__   )
									    and len(config.architectures   ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
															try:
																					_UpperCAmelCase									=  config.architectures[0]
																					_UpperCAmelCase									=  __import__("transformers"					,					fromlist=[model_class]   )
																					_UpperCAmelCase									=  getattr(snake_case__					,					snake_case__   )
																					_UpperCAmelCase									=  model_cls(snake_case__   )
															except ImportError:
																					raise ImportError(
																					    F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																					    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."   )
									else:
															_UpperCAmelCase									=  MODEL_WITH_LM_HEAD_MAPPING[config.__class__](snake_case__   )

									if self.args.torchscript:
															raise NotImplementedError("Training for torchscript is currently not implemented"   )
									else:
															_UpperCAmelCase									=  model

									model.train()
									model.to(self.args.device   )

									# encoder-decoder has vocab size saved differently
									_UpperCAmelCase									=  config.vocab_size if hasattr(snake_case__					,					"vocab_size"   ) else config.encoder.vocab_size
									_UpperCAmelCase									=  torch.randint(snake_case__					,					(batch_size, sequence_length)					,					dtype=torch.long					,					device=self.args.device   )

									if self.args.fpaa:
															logger.info("Running training in Mixed Precision..."   )
															if not self.args.is_gpu:
																					raise ValueError("Mixed precision is possible only for GPU."   )

															# amp seems to have memory leaks so that memory usage
															# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
															model.half()

									def compute_loss_and_backprob_encoder():
															_UpperCAmelCase									=  train_model(snake_case__					,					labels=snake_case__   )[0]
															loss.backward()
															return loss

									def compute_loss_and_backprob_encoder_decoder():
															_UpperCAmelCase									=  train_model(snake_case__					,					decoder_input_ids=snake_case__					,					labels=snake_case__   )[0]
															loss.backward()
															return loss

									_UpperCAmelCase									=  (
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train







			def UpperCamelCase    ( self					:    Any					,					snake_case__					:    Tuple   ):




									"""simple docstring"""


									try:
															if self.args.is_tpu or self.args.torchscript:
																					# run additional 10 times to stabilize compilation for tpu and torchscript
																					logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"   )
																					timeit.repeat(
																					    snake_case__					,					repeat=1					,					number=5					,					)

															# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
															_UpperCAmelCase									=  timeit.repeat(
															    snake_case__					,					repeat=self.args.repeat					,					number=10					,					)

															if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
																					import torch_xla.debug.metrics as met

																					self.print_fn(met.metrics_report()   )

															return min(snake_case__   ) / 10.0
									except RuntimeError as e:
															self.print_fn(F"""Doesn't fit on GPU. {e}"""   )
															return "N/A"







			def UpperCamelCase    ( self					:    Tuple					,					snake_case__					:    Callable[[], None]   ):




									"""simple docstring"""


									try:
															if self.args.trace_memory_line_by_line:
																					_UpperCAmelCase									=  start_memory_tracing("transformers"   )

															if self.args.is_tpu:
																					# tpu
																					raise NotImplementedError(
																					    "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"
																					    " `--no-memory` or `args.memory=False`"   )
															elif self.args.is_gpu:
																					if not is_pyanvml_available():
																											logger.warning(
																											    "py3nvml not installed, we won't log GPU memory usage. "
																											    "Install py3nvml (pip install py3nvml) to log information about GPU."   )
																											_UpperCAmelCase									=  "N/A"
																					else:
																											logger.info(
																											    "Measuring total GPU usage on GPU device. Make sure to not have additional processes running"
																											    " on the same GPU."   )
																											# init nvml
																											nvml.nvmlInit()
																											func()
																											_UpperCAmelCase									=  nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx   )
																											_UpperCAmelCase									=  nvml.nvmlDeviceGetMemoryInfo(snake_case__   )
																											_UpperCAmelCase									=  meminfo.used
																											_UpperCAmelCase									=  Memory(snake_case__   )
																											# shutdown nvml
																											nvml.nvmlShutdown()
															else:
																					# cpu
																					_UpperCAmelCase									=  measure_peak_memory_cpu(snake_case__   )
																					_UpperCAmelCase									=  Memory(snake_case__   ) if isinstance(snake_case__					,					snake_case__   ) else memory_bytes

															if self.args.trace_memory_line_by_line:
																					_UpperCAmelCase									=  stop_memory_tracing(snake_case__   )
															else:
																					_UpperCAmelCase									=  None

															return memory, summary
									except RuntimeError as e:
															self.print_fn(F"""Doesn't fit on GPU. {e}"""   )
															return "N/A", None




