




import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
       import torch


SCREAMING_SNAKE_CASE__ :   Optional[int]						=     logging.get_logger(__name__)




class 	UpperCamelCase__			:

    '''simple docstring'''



    def __init__(					self  ,						UpperCamelCase__  ,						UpperCamelCase__=21  ,						UpperCamelCase__=[]  ,						UpperCamelCase__=None  )  ->   str:
           lowerCamelCase     :   int    				=      model
           lowerCamelCase     :   int    				=      trace_batch_nums
           lowerCamelCase     :   str    				=      abort_after_batch_num

           # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
           lowerCamelCase     :   Any    				=      collections.deque([]  ,						UpperCamelCase__  )
           lowerCamelCase     :   Any    				=      []
           lowerCamelCase     :   int    				=      0
           lowerCamelCase     :   int    				=      0
           lowerCamelCase     :   Any    				=      False
           lowerCamelCase     :   int    				=      "                 "

           self.analyse_model()

           self.register_forward_hook()







    def 					_lowercase  (					self  ,						UpperCamelCase__=None  )  ->   Dict:
           if frame is not None:
                  self.expand_frame(UpperCamelCase__  )
           self.frames.append("\n".join(self.frame  )  )
           lowerCamelCase     :   Union[str, Any]    				=      []  # start a new frame







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   Union[str, Any]:
           self.frame.append(UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   Optional[Any]:
           print("\n".join(self.frames  )  )
           lowerCamelCase     :   Union[str, Any]    				=      []







    def 					_lowercase  (					self  )  ->   Any:
           lowerCamelCase     :   Dict    				=      []







    def 					_lowercase  (					self  )  ->   List[Any]:
           print(F'''\nDetected inf/nan during batch_number={self.batch_number}'''  )
           print(F'''Last {len(self.frames  )} forward frames:'''  )
           print(F'''{"abs min":8} {"abs max":8} metadata'''  )
           print("\n".join(self.frames  )  )
           print("\n\n"  )
           lowerCamelCase     :   Optional[int]    				=      []







    def 					_lowercase  (					self  )  ->   Dict:
           # extract the fully qualified module names, to be able to report at run time. e.g.:
           # encoder.block.2.layer.0.SelfAttention.o
           #
           # for shared weights only the first shared module name will be registered
           lowerCamelCase     :   int    				=      {m: name for name, m in self.model.named_modules()}
           # self.longest_module_name = max(len(v) for v in self.module_names.values())







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   List[Any]:
           if torch.is_tensor(UpperCamelCase__  ):
                  self.expand_frame(get_abs_min_max(UpperCamelCase__  ,						UpperCamelCase__  )  )
                  if detect_overflow(UpperCamelCase__  ,						UpperCamelCase__  ):
                         lowerCamelCase     :   Optional[Any]    				=      True
           elif var is None:
                  self.expand_frame(F'''{"None":>17} {ctx}'''  )
           else:
                  self.expand_frame(F'''{"not a tensor":>17} {ctx}'''  )







    def 					_lowercase  (					self  )  ->   Dict:
           self.expand_frame(F'''\n\n{self.prefix} *** Starting batch number={self.batch_number} ***'''  )
           self.expand_frame(F'''{"abs min":8} {"abs max":8} metadata'''  )







    def 					_lowercase  (					self  )  ->   Union[str, Any]:
           self.expand_frame(F'''{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n'''  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   int:
           self.expand_frame(F'''{self.prefix} {self.module_names[module]} {module.__class__.__name__}'''  )

           # params
           for name, p in module.named_parameters(recurse=UpperCamelCase__  ):
                  self.analyse_variable(UpperCamelCase__  ,						UpperCamelCase__  )

           # inputs
           if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                  for i, x in enumerate(UpperCamelCase__  ):
                         self.analyse_variable(UpperCamelCase__  ,						F'''input[{i}]'''  )
           else:
                  self.analyse_variable(UpperCamelCase__  ,						"input"  )

           # outputs
           if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                  for i, x in enumerate(UpperCamelCase__  ):
                         # possibly a tuple of tuples
                         if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                                for j, y in enumerate(UpperCamelCase__  ):
                                       self.analyse_variable(UpperCamelCase__  ,						F'''output[{i}][{j}]'''  )
                         else:
                                self.analyse_variable(UpperCamelCase__  ,						F'''output[{i}]'''  )
           else:
                  self.analyse_variable(UpperCamelCase__  ,						"output"  )

           self.save_frame()







    def 					_lowercase  (					self  )  ->   Dict:
           self.model.apply(self._register_forward_hook  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   Optional[Any]:
           module.register_forward_hook(self.forward_hook  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   int:
           # - input is a tuple of packed inputs (could be non-Tensors)
           # - output could be a Tensor or a tuple of Tensors and non-Tensors

           lowerCamelCase     :   str    				=      False

           lowerCamelCase     :   Any    				=      True if self.batch_number in self.trace_batch_nums else False
           if trace_mode:
                  self.reset_saved_frames()

           if self.total_calls == 0:
                  self.batch_start_frame()
           self.total_calls += 1

           # count batch numbers - the very first forward hook of the batch will be called when the
           # batch completes - i.e. it gets called very last - we know this batch has finished
           if module == self.model:
                  self.batch_number += 1
                  lowerCamelCase     :   Optional[Any]    				=      True

           self.create_frame(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )

           # if last_frame_of_batch:
           #     self.batch_end_frame()

           if trace_mode:
                  self.trace_frames()

           if last_frame_of_batch:
                  self.batch_start_frame()

           if self.detected_overflow and not trace_mode:
                  self.dump_saved_frames()

                  # now we can abort, as it's pointless to continue running
                  raise ValueError(
                      "DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. "
                      "Please scroll up above this traceback to see the activation values prior to this event."  )

           # abort after certain batch if requested to do so
           if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
                  raise ValueError(
                      F'''DebugUnderflowOverflow: aborting after {self.batch_number} batches due to'''
                      F''' `abort_after_batch_num={self.abort_after_batch_num}` arg'''  )




def   A   (   _SCREAMING_SNAKE_CASE				,_SCREAMING_SNAKE_CASE					)			->				List[str]:
       lowerCamelCase     :   Dict    				=      var.abs()
       return f'''{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}'''

def   A   (   _SCREAMING_SNAKE_CASE				,_SCREAMING_SNAKE_CASE					)			->				Union[str, Any]:
       lowerCamelCase     :   Any    				=      False
       if torch.isnan(_SCREAMING_SNAKE_CASE					).any().item():
              lowerCamelCase     :   Union[str, Any]    				=      True
              print(f'''{ctx} has nans'''					)
       if torch.isinf(_SCREAMING_SNAKE_CASE					).any().item():
              lowerCamelCase     :   List[Any]    				=      True
              print(f'''{ctx} has infs'''					)

       # if needed to monitor large elements can enable the following
       if 0:  # and detected:
              lowerCamelCase     :   Any    				=      var[torch.ge(var.abs()				,100					)]
              if naaa.numel() > 0:
                     print(f'''{ctx}:  n100={naaa.numel()}'''					)
              lowerCamelCase     :   Any    				=      var[torch.ge(var.abs()				,1000					)]
              if naaaa.numel() > 0:
                     print(f'''{ctx}: n1000={naaaa.numel()}'''					)
              lowerCamelCase     :   Any    				=      var[torch.ge(var.abs()				,1_0000					)]
              if naaaaa.numel() > 0:
                     print(f'''{ctx}: n10000={naaaaa.numel()}'''					)

       if 0:
              print(f'''min={var.min():9.2e} max={var.max():9.2e}'''					)

       if 0:
              print(f'''min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})'''					)

       return detected




class 	UpperCamelCase__			(lowerCAmelCase__  ):

    '''simple docstring'''



    lowerCamelCase_   :			Union[str, Any]			  =   """underflow_overflow"""
    lowerCamelCase_   :			Any			  =   """tpu_metrics_debug"""












import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
       import torch

       from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
       import pyanvml.pyanvml as nvml


SCREAMING_SNAKE_CASE__ :   List[Any]						=     logging.get_logger(__name__)




class 	UpperCamelCase__			(lowerCAmelCase__  ):

    '''simple docstring'''



    lowerCamelCase_   :			PyTorchBenchmarkArguments
    lowerCamelCase_   :			PretrainedConfig
    lowerCamelCase_   :			str			  =   "PyTorch"







    @property
    def 					_lowercase  (					self  )  ->   Optional[int]:
           return torch.__version__







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   float:
           lowerCamelCase     :   List[Any]    				=      self._prepare_inference_func(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )
           return self._measure_speed(_inference  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   [Memory, Optional[MemorySummary]]:
           lowerCamelCase     :   List[Any]    				=      self._prepare_inference_func(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )
           return self._measure_memory(_inference  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   float:
           lowerCamelCase     :   Dict    				=      self._prepare_train_func(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )
           return self._measure_speed(_train  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   [Memory, Optional[MemorySummary]]:
           lowerCamelCase     :   Optional[Any]    				=      self._prepare_train_func(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )
           return self._measure_memory(_train  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   Callable[[], None]:
           lowerCamelCase     :   Optional[Any]    				=      self.config_dict[model_name]

           if self.args.torchscript:
                  lowerCamelCase     :   Optional[Any]    				=      True

           lowerCamelCase     :   Dict    				=      (
               hasattr(UpperCamelCase__  ,						"architectures"  )
               and isinstance(config.architectures  ,						UpperCamelCase__  )
               and len(config.architectures  ) > 0
           )
           if not self.args.only_pretrain_model and has_model_class_in_config:
                  try:
                         lowerCamelCase     :   Optional[Any]    				=      config.architectures[0]
                         lowerCamelCase     :   Optional[int]    				=      __import__("transformers"  ,						fromlist=[model_class]  )
                         lowerCamelCase     :   Optional[int]    				=      getattr(UpperCamelCase__  ,						UpperCamelCase__  )
                         lowerCamelCase     :   List[str]    				=      model_cls(UpperCamelCase__  )
                  except ImportError:
                         raise ImportError(
                             F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                             " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
           else:
                  lowerCamelCase     :   Tuple    				=      MODEL_MAPPING[config.__class__](UpperCamelCase__  )

           model.eval()
           model.to(self.args.device  )

           # encoder-decoder has vocab size saved differently
           lowerCamelCase     :   str    				=      config.vocab_size if hasattr(UpperCamelCase__  ,						"vocab_size"  ) else config.encoder.vocab_size
           lowerCamelCase     :   int    				=      torch.randint(UpperCamelCase__  ,						(batch_size, sequence_length)  ,						dtype=torch.long  ,						device=self.args.device  )

           if self.args.fpaa:
                  logger.info("Running training in Mixed Precision..."  )
                  if not self.args.is_gpu:
                         raise ValueError("Mixed precision is possible only for GPU."  )
                  # amp seems to have memory leaks so that memory usage
                  # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                  model.half()

           if self.args.torchscript:
                  with torch.no_grad():
                         lowerCamelCase     :   Optional[int]    				=      torch.jit.trace(UpperCamelCase__  ,						UpperCamelCase__  )
           else:
                  lowerCamelCase     :   Dict    				=      model

           def encoder_decoder_forward():
                  with torch.no_grad():
                         lowerCamelCase     :   Dict    				=      inference_model(UpperCamelCase__  ,						decoder_input_ids=UpperCamelCase__  )
                  return outputs

           def encoder_forward():
                  with torch.no_grad():
                         lowerCamelCase     :   Union[str, Any]    				=      inference_model(UpperCamelCase__  )
                  return outputs

           lowerCamelCase     :   Dict    				=      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
           return _forward







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  )  ->   Callable[[], None]:
           lowerCamelCase     :   Optional[int]    				=      self.config_dict[model_name]

           lowerCamelCase     :   List[Any]    				=      (
               hasattr(UpperCamelCase__  ,						"architectures"  )
               and isinstance(config.architectures  ,						UpperCamelCase__  )
               and len(config.architectures  ) > 0
           )
           if not self.args.only_pretrain_model and has_model_class_in_config:
                  try:
                         lowerCamelCase     :   Union[str, Any]    				=      config.architectures[0]
                         lowerCamelCase     :   int    				=      __import__("transformers"  ,						fromlist=[model_class]  )
                         lowerCamelCase     :   Tuple    				=      getattr(UpperCamelCase__  ,						UpperCamelCase__  )
                         lowerCamelCase     :   Tuple    				=      model_cls(UpperCamelCase__  )
                  except ImportError:
                         raise ImportError(
                             F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                             " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
           else:
                  lowerCamelCase     :   List[str]    				=      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](UpperCamelCase__  )

           if self.args.torchscript:
                  raise NotImplementedError("Training for torchscript is currently not implemented"  )
           else:
                  lowerCamelCase     :   Dict    				=      model

           model.train()
           model.to(self.args.device  )

           # encoder-decoder has vocab size saved differently
           lowerCamelCase     :   Union[str, Any]    				=      config.vocab_size if hasattr(UpperCamelCase__  ,						"vocab_size"  ) else config.encoder.vocab_size
           lowerCamelCase     :   List[str]    				=      torch.randint(UpperCamelCase__  ,						(batch_size, sequence_length)  ,						dtype=torch.long  ,						device=self.args.device  )

           if self.args.fpaa:
                  logger.info("Running training in Mixed Precision..."  )
                  if not self.args.is_gpu:
                         raise ValueError("Mixed precision is possible only for GPU."  )

                  # amp seems to have memory leaks so that memory usage
                  # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                  model.half()

           def compute_loss_and_backprob_encoder():
                  lowerCamelCase     :   Tuple    				=      train_model(UpperCamelCase__  ,						labels=UpperCamelCase__  )[0]
                  loss.backward()
                  return loss

           def compute_loss_and_backprob_encoder_decoder():
                  lowerCamelCase     :   Optional[int]    				=      train_model(UpperCamelCase__  ,						decoder_input_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  )[0]
                  loss.backward()
                  return loss

           lowerCamelCase     :   Union[str, Any]    				=      (
               compute_loss_and_backprob_encoder_decoder
               if config.is_encoder_decoder
               else compute_loss_and_backprob_encoder
           )
           return _train







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   float:
           try:
                  if self.args.is_tpu or self.args.torchscript:
                         # run additional 10 times to stabilize compilation for tpu and torchscript
                         logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"  )
                         timeit.repeat(
                             UpperCamelCase__  ,						repeat=1  ,						number=5  ,						)

                  # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                  lowerCamelCase     :   List[str]    				=      timeit.repeat(
                      UpperCamelCase__  ,						repeat=self.args.repeat  ,						number=10  ,						)

                  if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
                         import torch_xla.debug.metrics as met

                         self.print_fn(met.metrics_report()  )

                  return min(UpperCamelCase__  ) / 10.0
           except RuntimeError as e:
                  self.print_fn(F'''Doesn\'t fit on GPU. {e}'''  )
                  return "N/A"







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   [Memory, MemorySummary]:
           try:
                  if self.args.trace_memory_line_by_line:
                         lowerCamelCase     :   str    				=      start_memory_tracing("transformers"  )

                  if self.args.is_tpu:
                         # tpu
                         raise NotImplementedError(
                             "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"
                             " `--no-memory` or `args.memory=False`"  )
                  elif self.args.is_gpu:
                         if not is_pyanvml_available():
                                logger.warning(
                                    "py3nvml not installed, we won't log GPU memory usage. "
                                    "Install py3nvml (pip install py3nvml) to log information about GPU."  )
                                lowerCamelCase     :   Optional[Any]    				=      "N/A"
                         else:
                                logger.info(
                                    "Measuring total GPU usage on GPU device. Make sure to not have additional processes running"
                                    " on the same GPU."  )
                                # init nvml
                                nvml.nvmlInit()
                                func()
                                lowerCamelCase     :   int    				=      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
                                lowerCamelCase     :   Dict    				=      nvml.nvmlDeviceGetMemoryInfo(UpperCamelCase__  )
                                lowerCamelCase     :   int    				=      meminfo.used
                                lowerCamelCase     :   Dict    				=      Memory(UpperCamelCase__  )
                                # shutdown nvml
                                nvml.nvmlShutdown()
                  else:
                         # cpu
                         lowerCamelCase     :   Union[str, Any]    				=      measure_peak_memory_cpu(UpperCamelCase__  )
                         lowerCamelCase     :   int    				=      Memory(UpperCamelCase__  ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ) else memory_bytes

                  if self.args.trace_memory_line_by_line:
                         lowerCamelCase     :   List[str]    				=      stop_memory_tracing(UpperCamelCase__  )
                  else:
                         lowerCamelCase     :   int    				=      None

                  return memory, summary
           except RuntimeError as e:
                  self.print_fn(F'''Doesn\'t fit on GPU. {e}'''  )
                  return "N/A", None






