

'''simple docstring'''




import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


_a				: Any 							=						logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

_a				: Optional[int] 							=						{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
_a				: List[Any] 							=						{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
_a				: Union[str, Any] 							=						sorted(arg_to_scheduler.keys())
_a				: List[Any] 							=						'{' + ', '.join(arg_to_scheduler_choices) + '}'



class        lowercase_	(		pl.LightningModule ):

  '''simple docstring'''





  def __init__(     self   , a_   , a_=None   , a_="base"   , a_=None   , a_=None   , a_=None   , **a_   , )							->				List[Any]:


    """simple docstring"""
    super().__init__()
    # TODO: move to self.save_hyperparameters()
    # self.save_hyperparameters()
    # can also expand arguments into trainer signature for easier reading

    self.save_hyperparameters(a_     )
    UpperCAmelCase													=	0
    UpperCAmelCase													=	Path(self.hparams.output_dir     )
    UpperCAmelCase													=	self.hparams.cache_dir if self.hparams.cache_dir else None
    if config is None:
      UpperCAmelCase													=	AutoConfig.from_pretrained(
          self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   , **({'num_labels': num_labels} if num_labels is not None else {})   , cache_dir=a_   , **a_   , )
    else:
      UpperCAmelCase													=	config

    UpperCAmelCase													=	('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
    for p in extra_model_params:
      if getattr(self.hparams   , a_   , a_     ):
        assert hasattr(self.config   , a_     ), F'''model config doesn\'t have a `{p}` attribute'''
        setattr(self.config   , a_   , getattr(self.hparams   , a_     )     )

    if tokenizer is None:
      UpperCAmelCase													=	AutoTokenizer.from_pretrained(
          self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   , cache_dir=a_   , )
    else:
      UpperCAmelCase													=	tokenizer
    UpperCAmelCase													=	MODEL_MODES[mode]
    if model is None:
      UpperCAmelCase													=	self.model_type.from_pretrained(
          self.hparams.model_name_or_path   , from_tf=bool('.ckpt' in self.hparams.model_name_or_path     )   , config=self.config   , cache_dir=a_   , )
    else:
      UpperCAmelCase													=	model
  def 						snake_case_			(     self   , *a_   , **a_     )							->				List[str]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_type.from_pretrained(*a_   , **a_     )
  def 						snake_case_			(     self     )							->				Any:


    """simple docstring"""
    UpperCAmelCase													=	arg_to_scheduler[self.hparams.lr_scheduler]
    UpperCAmelCase													=	get_schedule_func(
        self.opt   , num_warmup_steps=self.hparams.warmup_steps   , num_training_steps=self.total_steps()     )
    UpperCAmelCase													=	{'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
    return scheduler
  def 						snake_case_			(     self     )							->				Dict:


    """simple docstring"""
    UpperCAmelCase													=	self.model
    UpperCAmelCase													=	['bias', 'LayerNorm.weight']
    UpperCAmelCase													=	[
        {
            'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay     )],
            'weight_decay': self.hparams.weight_decay,
        },
        {
            'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay     )],
            'weight_decay': 0.0,
        },
    ]
    if self.hparams.adafactor:
      UpperCAmelCase													=	Adafactor(
          a_   , lr=self.hparams.learning_rate   , scale_parameter=a_   , relative_step=a_     )

    else:
      UpperCAmelCase													=	AdamW(
          a_   , lr=self.hparams.learning_rate   , eps=self.hparams.adam_epsilon     )
    UpperCAmelCase													=	optimizer

    UpperCAmelCase													=	self.get_lr_scheduler()

    return [optimizer], [scheduler]
  def 						snake_case_			(     self   , a_   , a_     )							->				int:


    """simple docstring"""
    return self.validation_step(a_   , a_     )
  def 						snake_case_			(     self   , a_     )							->				Tuple:


    """simple docstring"""
    return self.validation_end(a_     )
  def 						snake_case_			(     self     )							->				int:


    """simple docstring"""
    UpperCAmelCase													=	max(1   , self.hparams.gpus     )  # TODO: consider num_tpu_cores
    UpperCAmelCase													=	self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
    return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
  def 						snake_case_			(     self   , a_     )							->				Optional[Any]:


    """simple docstring"""
    if stage == "test":
      UpperCAmelCase													=	len(self.test_dataloader().dataset     )
    else:
      UpperCAmelCase													=	self.get_dataloader('train'   , self.hparams.train_batch_size   , shuffle=a_     )
      UpperCAmelCase													=	len(self.train_dataloader().dataset     )
  def 						snake_case_			(     self   , a_   , a_   , a_ = False     )							->				Tuple:


    """simple docstring"""
    raise NotImplementedError('You must implement this for your task'     )
  def 						snake_case_			(     self     )							->				Dict:


    """simple docstring"""
    return self.train_loader
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    return self.get_dataloader('dev'   , self.hparams.eval_batch_size   , shuffle=a_     )
  def 						snake_case_			(     self     )							->				int:


    """simple docstring"""
    return self.get_dataloader('test'   , self.hparams.eval_batch_size   , shuffle=a_     )
  def 						snake_case_			(     self   , a_     )							->				str:


    """simple docstring"""
    return os.path.join(
        self.hparams.data_dir   , 'cached_{}_{}_{}'.format(
            a_   , list(filter(a_   , self.hparams.model_name_or_path.split('/'     )     )     ).pop()   , str(self.hparams.max_seq_length     )   , )   , )
  @pl.utilities.rank_zero_only
  def 						snake_case_			(     self   , a_     )							->				None:


    """simple docstring"""
    UpperCAmelCase													=	self.output_dir.joinpath('best_tfmr'     )
    UpperCAmelCase													=	self.step_count
    self.model.save_pretrained(a_     )
    self.tokenizer.save_pretrained(a_     )





  @staticmethod
  def 						snake_case_			(     a_   , a_     )							->				List[str]:


    """simple docstring"""
    parser.add_argument(
        '--model_name_or_path'   , default=a_   , type=a_   , required=a_   , help='Path to pretrained model or model identifier from huggingface.co/models'   , )
    parser.add_argument(
        '--config_name'   , default=''   , type=a_   , help='Pretrained config name or path if not the same as model_name'     )
    parser.add_argument(
        '--tokenizer_name'   , default=a_   , type=a_   , help='Pretrained tokenizer name or path if not the same as model_name'   , )
    parser.add_argument(
        '--cache_dir'   , default=''   , type=a_   , help='Where do you want to store the pre-trained models downloaded from huggingface.co'   , )
    parser.add_argument(
        '--encoder_layerdrop'   , type=a_   , help='Encoder layer dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--decoder_layerdrop'   , type=a_   , help='Decoder layer dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--dropout'   , type=a_   , help='Dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--attention_dropout'   , type=a_   , help='Attention dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument('--learning_rate'   , default=5E-5   , type=a_   , help='The initial learning rate for Adam.'     )
    parser.add_argument(
        '--lr_scheduler'   , default='linear'   , choices=a_   , metavar=a_   , type=a_   , help='Learning rate scheduler'   , )
    parser.add_argument('--weight_decay'   , default=0.0   , type=a_   , help='Weight decay if we apply some.'     )
    parser.add_argument('--adam_epsilon'   , default=1E-8   , type=a_   , help='Epsilon for Adam optimizer.'     )
    parser.add_argument('--warmup_steps'   , default=0   , type=a_   , help='Linear warmup over warmup_steps.'     )
    parser.add_argument('--num_workers'   , default=4   , type=a_   , help='kwarg passed to DataLoader'     )
    parser.add_argument('--num_train_epochs'   , dest='max_epochs'   , default=3   , type=a_     )
    parser.add_argument('--train_batch_size'   , default=3_2   , type=a_     )
    parser.add_argument('--eval_batch_size'   , default=3_2   , type=a_     )
    parser.add_argument('--adafactor'   , action='store_true'     )




class        lowercase_	(		pl.Callback ):

  '''simple docstring'''





  def 						snake_case_			(     self   , a_   , a_     )							->				Union[str, Any]:


    """simple docstring"""
    if (
        trainer.is_global_zero and trainer.global_rank == 0
    ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
      pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.




class        lowercase_	(		pl.Callback ):

  '''simple docstring'''





  def 						snake_case_			(     self   , a_   , a_     )							->				str:


    """simple docstring"""
    UpperCAmelCase													=	trainer.lr_schedulers[0]['scheduler']
    UpperCAmelCase													=	{F'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()     )}
    pl_module.logger.log_metrics(a_     )
  def 						snake_case_			(     self   , a_   , a_     )							->				List[Any]:


    """simple docstring"""
    rank_zero_info('***** Validation results *****'     )
    UpperCAmelCase													=	trainer.callback_metrics
    # Log results
    for key in sorted(a_     ):
      if key not in ["log", "progress_bar"]:
        rank_zero_info('{} = {}\n'.format(a_   , str(metrics[key]     )     )     )





  def 						snake_case_			(     self   , a_   , a_     )							->				str:


    """simple docstring"""
    rank_zero_info('***** Test results *****'     )
    UpperCAmelCase													=	trainer.callback_metrics
    # Log and save results to file
    UpperCAmelCase													=	os.path.join(pl_module.hparams.output_dir   , 'test_results.txt'     )
    with open(a_   , 'w'     ) as writer:
      for key in sorted(a_     ):
        if key not in ["log", "progress_bar"]:
          rank_zero_info('{} = {}\n'.format(a_   , str(metrics[key]     )     )     )
          writer.write('{} = {}\n'.format(a_   , str(metrics[key]     )     )     )




def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Dict       ,   SCREAMING_SNAKE_CASE      :   Union[str, Any]						):
  #  To allow all pl args uncomment the following line
  #  parser = pl.Trainer.add_argparse_args(parser)
  parser.add_argument(
      '--output_dir'       ,   default=SCREAMING_SNAKE_CASE       ,   type=SCREAMING_SNAKE_CASE       ,   required=SCREAMING_SNAKE_CASE       ,   help='The output directory where the model predictions and checkpoints will be written.'       ,   )
  parser.add_argument(
      '--fp16'       ,   action='store_true'       ,   help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'       ,   )

  parser.add_argument(
      '--fp16_opt_level'       ,   type=SCREAMING_SNAKE_CASE       ,   default='O2'       ,   help=(
          'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
          'See details at https://nvidia.github.io/apex/amp.html'
      )       ,   )
  parser.add_argument('--n_tpu_cores'       ,   dest='tpu_cores'       ,   type=SCREAMING_SNAKE_CASE						)
  parser.add_argument('--max_grad_norm'       ,   dest='gradient_clip_val'       ,   default=1.0       ,   type=SCREAMING_SNAKE_CASE       ,   help='Max gradient norm'						)
  parser.add_argument('--do_train'       ,   action='store_true'       ,   help='Whether to run training.'						)
  parser.add_argument('--do_predict'       ,   action='store_true'       ,   help='Whether to run predictions on the test set.'						)
  parser.add_argument(
      '--gradient_accumulation_steps'       ,   dest='accumulate_grad_batches'       ,   type=SCREAMING_SNAKE_CASE       ,   default=1       ,   help='Number of updates steps to accumulate before performing a backward/update pass.'       ,   )
  parser.add_argument('--seed'       ,   type=SCREAMING_SNAKE_CASE       ,   default=42       ,   help='random seed for initialization'						)
  parser.add_argument(
      '--data_dir'       ,   default=SCREAMING_SNAKE_CASE       ,   type=SCREAMING_SNAKE_CASE       ,   required=SCREAMING_SNAKE_CASE       ,   help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'       ,   )



def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   BaseTransformer       ,   SCREAMING_SNAKE_CASE      :   argparse.Namespace       ,   SCREAMING_SNAKE_CASE      :   int=None       ,   SCREAMING_SNAKE_CASE      :   Union[str, Any]=True       ,   SCREAMING_SNAKE_CASE      :   Optional[int]=None       ,   SCREAMING_SNAKE_CASE      :   int=[]       ,   SCREAMING_SNAKE_CASE      :   Union[str, Any]=None       ,   SCREAMING_SNAKE_CASE      :   int=None       ,   **SCREAMING_SNAKE_CASE      :   str       ,   ):
  pl.seed_everything(args.seed						)

  # init model
  UpperCAmelCase													=	Path(model.hparams.output_dir						)
  odir.mkdir(exist_ok=SCREAMING_SNAKE_CASE						)

  # add custom checkpoints
  if checkpoint_callback is None:
    UpperCAmelCase													=	pl.callbacks.ModelCheckpoint(
        filepath=args.output_dir       ,   prefix='checkpoint'       ,   monitor='val_loss'       ,   mode='min'       ,   save_top_k=1						)
  if early_stopping_callback:
    extra_callbacks.append(SCREAMING_SNAKE_CASE						)
  if logging_callback is None:
    UpperCAmelCase													=	LoggingCallback()

  UpperCAmelCase													=	{}

  # TODO: remove with PyTorch 1.6 since pl uses native amp
  if args.fpaa:
    UpperCAmelCase													=	16
    # train_params["amp_level"] = args.fp16_opt_level

  if args.gpus > 1:
    UpperCAmelCase													=	'auto'  # "ddp"
    UpperCAmelCase													=	'ddp'

  UpperCAmelCase													=	args.accumulate_grad_batches
  UpperCAmelCase													=	None  # extra_train_kwargs.get("profiler", None) #get unwanted logs
  UpperCAmelCase													=	'auto'

  UpperCAmelCase													=	pl.Trainer.from_argparse_args(
      SCREAMING_SNAKE_CASE       ,   weights_summary=SCREAMING_SNAKE_CASE       ,   callbacks=[logging_callback] + extra_callbacks + [checkpoint_callback] + [InitCallback()]       ,   logger=SCREAMING_SNAKE_CASE       ,   **SCREAMING_SNAKE_CASE       ,   )

  if args.do_train:
    trainer.fit(SCREAMING_SNAKE_CASE						)

  return trainer





'''simple docstring'''




from typing import List, Optional, Tuple, Union

from . import logging
from .import_utils import is_torch_available, is_torch_version


if is_torch_available():
  import torch

_a				: List[Any] 							=						logging.get_logger(__name__)  # pylint: disable=invalid-name

try:
  from torch._dynamo import allow_in_graph as maybe_allow_in_graph
except (ImportError, ModuleNotFoundError):


  def       lowerCamelCase__		(						cls      :   Dict						):
    return cls


def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Union[Tuple, List]       ,   SCREAMING_SNAKE_CASE      :   Optional[Union[List["torch.Generator"], "torch.Generator"]] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.device"] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.dtype"] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.layout"] = None       ,   ):
  UpperCAmelCase													=	device
  UpperCAmelCase													=	shape[0]

  UpperCAmelCase													=	layout or torch.strided
  UpperCAmelCase													=	device or torch.device('cpu'						)

  if generator is not None:
    UpperCAmelCase													=	generator.device.type if not isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						) else generator[0].device.type
    if gen_device_type != device.type and gen_device_type == "cpu":
      UpperCAmelCase													=	'cpu'
      if device != "mps":
        logger.info(
            f'''The passed generator was created on \'cpu\' even though a tensor on {device} was expected.'''
            f''' Tensors will be created on \'cpu\' and then moved to {device}. Note that one can probably'''
            f''' slighly speed up this function by passing a generator that was created on the {device} device.'''						)
    elif gen_device_type != device.type and gen_device_type == "cuda":
      raise ValueError(f'''Cannot generate a {device} tensor from a generator of type {gen_device_type}.'''						)

  if isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						):
    UpperCAmelCase													=	(1,) + shape[1:]
    UpperCAmelCase													=	[
        torch.randn(SCREAMING_SNAKE_CASE       ,   generator=generator[i]       ,   device=SCREAMING_SNAKE_CASE       ,   dtype=SCREAMING_SNAKE_CASE       ,   layout=SCREAMING_SNAKE_CASE						)
        for i in range(SCREAMING_SNAKE_CASE						)
    ]
    UpperCAmelCase													=	torch.cat(SCREAMING_SNAKE_CASE       ,   dim=0						).to(SCREAMING_SNAKE_CASE						)
  else:
    UpperCAmelCase													=	torch.randn(SCREAMING_SNAKE_CASE       ,   generator=SCREAMING_SNAKE_CASE       ,   device=SCREAMING_SNAKE_CASE       ,   dtype=SCREAMING_SNAKE_CASE       ,   layout=SCREAMING_SNAKE_CASE						).to(SCREAMING_SNAKE_CASE						)

  return latents



def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Tuple						):
  if is_torch_version('<'       ,   '2.0.0'						) or not hasattr(SCREAMING_SNAKE_CASE       ,   '_dynamo'						):
    return False
  return isinstance(SCREAMING_SNAKE_CASE       ,   torch._dynamo.eval_frame.OptimizedModule						)


