





"""simple docstring"""
import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
    import torch

    from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
    import pyanvml.pyanvml as nvml


lowerCAmelCase:	Any							      =logging.get_logger(__name__)
class lowerCamelCase__  (    __UpperCAmelCase				):
   __UpperCAmelCase        =					42
   __UpperCAmelCase        =					42
   __UpperCAmelCase        =					"PyTorch"



   @property
   def    _UpperCAmelCase	(       self	) ->    Union[str, Any]:







      """simple docstring"""
      return torch.__version__



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    float:







      """simple docstring"""
      lowercase    :			Any  =       self._prepare_inference_func(lowerCamelCase__      ,   lowerCamelCase__      ,   lowerCamelCase__	)
      return self._measure_speed(_inference	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    [Memory, Optional[MemorySummary]]:







      """simple docstring"""
      lowercase    :			List[str]  =       self._prepare_inference_func(lowerCamelCase__      ,   lowerCamelCase__      ,   lowerCamelCase__	)
      return self._measure_memory(_inference	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    float:







      """simple docstring"""
      lowercase    :			List[Any]  =       self._prepare_train_func(lowerCamelCase__      ,   lowerCamelCase__      ,   lowerCamelCase__	)
      return self._measure_speed(_train	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    [Memory, Optional[MemorySummary]]:







      """simple docstring"""
      lowercase    :			Union[str, Any]  =       self._prepare_train_func(lowerCamelCase__      ,   lowerCamelCase__      ,   lowerCamelCase__	)
      return self._measure_memory(_train	)



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    Callable[[], None]:







      """simple docstring"""
      lowercase    :			int  =       self.config_dict[model_name]

      if self.args.torchscript:
         lowercase    :			Dict  =       True

      lowercase    :			List[Any]  =       (
          hasattr(lowerCamelCase__      ,   """architectures"""	)
          and isinstance(config.architectures      ,   lowerCamelCase__	)
          and len(config.architectures	) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
         try:
            lowercase    :			List[Any]  =       config.architectures[0]
            lowercase    :			str  =       __import__("""transformers"""      ,   fromlist=[model_class]	)
            lowercase    :			Union[str, Any]  =       getattr(lowerCamelCase__      ,   lowerCamelCase__	)
            lowercase    :			Tuple  =       model_cls(lowerCamelCase__	)
         except ImportError:
            raise ImportError(
                f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""	)
      else:
         lowercase    :			Optional[int]  =       MODEL_MAPPING[config.__class__](lowerCamelCase__	)

      model.eval()
      model.to(self.args.device	)

      # encoder-decoder has vocab size saved differently
      lowercase    :			List[Any]  =       config.vocab_size if hasattr(lowerCamelCase__      ,   """vocab_size"""	) else config.encoder.vocab_size
      lowercase    :			Optional[Any]  =       torch.randint(lowerCamelCase__      ,   (batch_size, sequence_length)      ,   dtype=torch.long      ,   device=self.args.device	)

      if self.args.fpaa:
         logger.info("""Running training in Mixed Precision..."""	)
         if not self.args.is_gpu:
            raise ValueError("""Mixed precision is possible only for GPU."""	)
         # amp seems to have memory leaks so that memory usage
         # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
         model.half()

      if self.args.torchscript:
         with torch.no_grad():
            lowercase    :			Optional[int]  =       torch.jit.trace(lowerCamelCase__      ,   lowerCamelCase__	)
      else:
         lowercase    :			int  =       model

      def encoder_decoder_forward():
         with torch.no_grad():
            lowercase    :			Tuple  =       inference_model(lowerCamelCase__      ,   decoder_input_ids=lowerCamelCase__	)
         return outputs

      def encoder_forward():
         with torch.no_grad():
            lowercase    :			Any  =       inference_model(lowerCamelCase__	)
         return outputs

      lowercase    :			Any  =       encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
      return _forward



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case      ,   snake_case	) ->    Callable[[], None]:







      """simple docstring"""
      lowercase    :			Union[str, Any]  =       self.config_dict[model_name]

      lowercase    :			List[Any]  =       (
          hasattr(lowerCamelCase__      ,   """architectures"""	)
          and isinstance(config.architectures      ,   lowerCamelCase__	)
          and len(config.architectures	) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
         try:
            lowercase    :			Optional[int]  =       config.architectures[0]
            lowercase    :			str  =       __import__("""transformers"""      ,   fromlist=[model_class]	)
            lowercase    :			List[Any]  =       getattr(lowerCamelCase__      ,   lowerCamelCase__	)
            lowercase    :			List[str]  =       model_cls(lowerCamelCase__	)
         except ImportError:
            raise ImportError(
                f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""	)
      else:
         lowercase    :			Optional[int]  =       MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCamelCase__	)

      if self.args.torchscript:
         raise NotImplementedError("""Training for torchscript is currently not implemented"""	)
      else:
         lowercase    :			Optional[Any]  =       model

      model.train()
      model.to(self.args.device	)

      # encoder-decoder has vocab size saved differently
      lowercase    :			List[Any]  =       config.vocab_size if hasattr(lowerCamelCase__      ,   """vocab_size"""	) else config.encoder.vocab_size
      lowercase    :			List[str]  =       torch.randint(lowerCamelCase__      ,   (batch_size, sequence_length)      ,   dtype=torch.long      ,   device=self.args.device	)

      if self.args.fpaa:
         logger.info("""Running training in Mixed Precision..."""	)
         if not self.args.is_gpu:
            raise ValueError("""Mixed precision is possible only for GPU."""	)

         # amp seems to have memory leaks so that memory usage
         # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
         model.half()

      def compute_loss_and_backprob_encoder():
         lowercase    :			str  =       train_model(lowerCamelCase__      ,   labels=lowerCamelCase__	)[0]
         loss.backward()
         return loss

      def compute_loss_and_backprob_encoder_decoder():
         lowercase    :			Any  =       train_model(lowerCamelCase__      ,   decoder_input_ids=lowerCamelCase__      ,   labels=lowerCamelCase__	)[0]
         loss.backward()
         return loss

      lowercase    :			int  =       (
          compute_loss_and_backprob_encoder_decoder
          if config.is_encoder_decoder
          else compute_loss_and_backprob_encoder
      )
      return _train



   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    float:







      """simple docstring"""



      try:
         if self.args.is_tpu or self.args.torchscript:
            # run additional 10 times to stabilize compilation for tpu and torchscript
            logger.info("""Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"""	)
            timeit.repeat(
                lowerCamelCase__      ,   repeat=1      ,   number=5      ,   )

         # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
         lowercase    :			Tuple  =       timeit.repeat(
             lowerCamelCase__      ,   repeat=self.args.repeat      ,   number=1_0      ,   )

         if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
            import torch_xla.debug.metrics as met

            self.print_fn(met.metrics_report()	)

         return min(lowerCamelCase__	) / 10.0
      except RuntimeError as e:
         self.print_fn(f'''Doesn\'t fit on GPU. {e}'''	)
         return "N/A"





   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    [Memory, MemorySummary]:







      """simple docstring"""



      try:
         if self.args.trace_memory_line_by_line:
            lowercase    :			Dict  =       start_memory_tracing("""transformers"""	)

         if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"""
                """ `--no-memory` or `args.memory=False`"""	)
         elif self.args.is_gpu:
            if not is_pyanvml_available():
               logger.warning(
                   """py3nvml not installed, we won't log GPU memory usage. """
                   """Install py3nvml (pip install py3nvml) to log information about GPU."""	)
               lowercase    :			Optional[int]  =       """N/A"""
            else:
               logger.info(
                   """Measuring total GPU usage on GPU device. Make sure to not have additional processes running"""
                   """ on the same GPU."""	)
               # init nvml
               nvml.nvmlInit()
               func()
               lowercase    :			Dict  =       nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx	)
               lowercase    :			Optional[int]  =       nvml.nvmlDeviceGetMemoryInfo(lowerCamelCase__	)
               lowercase    :			Union[str, Any]  =       meminfo.used
               lowercase    :			Union[str, Any]  =       Memory(lowerCamelCase__	)
               # shutdown nvml
               nvml.nvmlShutdown()
         else:
            # cpu
            lowercase    :			Tuple  =       measure_peak_memory_cpu(lowerCamelCase__	)
            lowercase    :			Any  =       Memory(lowerCamelCase__	) if isinstance(lowerCamelCase__      ,   lowerCamelCase__	) else memory_bytes

         if self.args.trace_memory_line_by_line:
            lowercase    :			Optional[Any]  =       stop_memory_tracing(lowerCamelCase__	)
         else:
            lowercase    :			Union[str, Any]  =       None

         return memory, summary
      except RuntimeError as e:
         self.print_fn(f'''Doesn\'t fit on GPU. {e}'''	)
         return "N/A", None









"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase:	int							      =logging.get_logger(__name__)

lowerCAmelCase:	int							      ={
    "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/config.json",
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}
class lowerCamelCase__  (    __UpperCamelCase				):
   __UpperCAmelCase        =					"""qdqbert"""



   def __init__(       self      ,   snake_case=3_0_5_2_2      ,   snake_case=7_6_8      ,   snake_case=1_2      ,   snake_case=1_2      ,   snake_case=3_0_7_2      ,   snake_case="gelu"      ,   snake_case=0.1      ,   snake_case=0.1      ,   snake_case=5_1_2      ,   snake_case=2      ,   snake_case=0.02      ,   snake_case=1E-12      ,   snake_case=True      ,   snake_case=1      ,   snake_case=0      ,   snake_case=2      ,   **snake_case      ,   ) ->    List[str]:







      """simple docstring"""
      super().__init__(pad_token_id=snake_case      ,   bos_token_id=snake_case      ,   eos_token_id=snake_case      ,   **snake_case	)

      lowercase    :			Optional[Any]  =       vocab_size
      lowercase    :			int  =       max_position_embeddings
      lowercase    :			Dict  =       hidden_size
      lowercase    :			Union[str, Any]  =       num_hidden_layers
      lowercase    :			str  =       num_attention_heads
      lowercase    :			List[str]  =       intermediate_size
      lowercase    :			List[str]  =       hidden_act
      lowercase    :			List[str]  =       hidden_dropout_prob
      lowercase    :			Optional[Any]  =       attention_probs_dropout_prob
      lowercase    :			Optional[Any]  =       initializer_range
      lowercase    :			str  =       type_vocab_size
      lowercase    :			Optional[int]  =       layer_norm_eps
      lowercase    :			Optional[int]  =       use_cache


