






'''simple docstring'''
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


__lowerCAmelCase							=logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

__lowerCAmelCase							={
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
__lowerCAmelCase							={
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
__lowerCAmelCase							=sorted(arg_to_scheduler.keys())
__lowerCAmelCase							="{" + ", ".join(arg_to_scheduler_choices) + "}"


class    _snake_case   (				pl.LightningModule					):

					"""simple docstring"""





					def __init__(     self   , UpperCAmelCase__   , UpperCAmelCase__=None   , UpperCAmelCase__="base"   , UpperCAmelCase__=None   , UpperCAmelCase__=None   , UpperCAmelCase__=None   , **UpperCAmelCase__   , )      ->	List[str]:
										super().__init__()
										# TODO: move to self.save_hyperparameters()
										# self.save_hyperparameters()
										# can also expand arguments into trainer signature for easier reading

										self.save_hyperparameters(UpperCAmelCase__     )
										a_									=   0
										a_									=   Path(self.hparams.output_dir     )
										a_									=   self.hparams.cache_dir if self.hparams.cache_dir else None
										if config is None:
															a_									=   AutoConfig.from_pretrained(
															    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   , **({'num_labels': num_labels} if num_labels is not None else {})   , cache_dir=UpperCAmelCase__   , **UpperCAmelCase__   , )
										else:
															a_									=   config

										a_									=   ('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
										for p in extra_model_params:
															if getattr(self.hparams   , UpperCAmelCase__   , UpperCAmelCase__     ):
																				assert hasattr(self.config   , UpperCAmelCase__     ), F'''model config doesn\'t have a `{p}` attribute'''
																				setattr(self.config   , UpperCAmelCase__   , getattr(self.hparams   , UpperCAmelCase__     )     )

										if tokenizer is None:
															a_									=   AutoTokenizer.from_pretrained(
															    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   , cache_dir=UpperCAmelCase__   , )
										else:
															a_									=   tokenizer
										a_									=   MODEL_MODES[mode]
										if model is None:
															a_									=   self.model_type.from_pretrained(
															    self.hparams.model_name_or_path   , from_tf=bool('.ckpt' in self.hparams.model_name_or_path     )   , config=self.config   , cache_dir=UpperCAmelCase__   , )
										else:
															a_									=   model




					def  __SCREAMING_SNAKE_CASE (     self   , *UpperCAmelCase__   , **UpperCAmelCase__     )      ->	Any:
										a_									=   self.model_type.from_pretrained(*UpperCAmelCase__   , **UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	Optional[int]:
										a_									=   arg_to_scheduler[self.hparams.lr_scheduler]
										a_									=   get_schedule_func(
										    self.opt   , num_warmup_steps=self.hparams.warmup_steps   , num_training_steps=self.total_steps()     )
										a_									=   {'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
										return scheduler




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	Optional[Any]:
										a_									=   self.model
										a_									=   ['bias', 'LayerNorm.weight']
										a_									=   [
										    {
										        'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay     )],
										        'weight_decay': self.hparams.weight_decay,
										    },
										    {
										        'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay     )],
										        'weight_decay': 0.0,
										    },
										]
										if self.hparams.adafactor:
															a_									=   Adafactor(
															    UpperCAmelCase__   , lr=self.hparams.learning_rate   , scale_parameter=UpperCAmelCase__   , relative_step=UpperCAmelCase__     )

										else:
															a_									=   AdamW(
															    UpperCAmelCase__   , lr=self.hparams.learning_rate   , eps=self.hparams.adam_epsilon     )
										a_									=   optimizer

										a_									=   self.get_lr_scheduler()

										return [optimizer], [scheduler]




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__   , UpperCAmelCase__     )      ->	Tuple:
										return self.validation_step(UpperCAmelCase__   , UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__     )      ->	Tuple:
										return self.validation_end(UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	int:
										a_									=   max(1   , self.hparams.gpus     )  # TODO: consider num_tpu_cores
										a_									=   self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
										return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__     )      ->	List[Any]:
										if mode == "test":
															a_									=   len(self.test_dataloader().dataset     )
										else:
															a_									=   self.get_dataloader('train'   , self.hparams.train_batch_size   , shuffle=UpperCAmelCase__     )
															a_									=   len(self.train_dataloader().dataset     )




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__   , UpperCAmelCase__   , UpperCAmelCase__ = False     )      ->	int:
										raise NotImplementedError('You must implement this for your task'     )




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	Any:
										return self.train_loader




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	Optional[Any]:
										return self.get_dataloader('dev'   , self.hparams.eval_batch_size   , shuffle=UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self     )      ->	str:
										return self.get_dataloader('test'   , self.hparams.eval_batch_size   , shuffle=UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__     )      ->	List[str]:
										return os.path.join(
										    self.hparams.data_dir   , 'cached_{}_{}_{}'.format(
										        UpperCAmelCase__   , list(filter(UpperCAmelCase__   , self.hparams.model_name_or_path.split('/'     )     )     ).pop()   , str(self.hparams.max_seq_length     )   , )   , )




					@pl.utilities.rank_zero_only
					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__     )      ->	None:
										a_									=   self.output_dir.joinpath('best_tfmr'     )
										a_									=   self.step_count
										self.model.save_pretrained(UpperCAmelCase__     )
										self.tokenizer.save_pretrained(UpperCAmelCase__     )




					@staticmethod
					def  __SCREAMING_SNAKE_CASE (     UpperCAmelCase__   , UpperCAmelCase__     )      ->	Optional[int]:
										parser.add_argument(
										    '--model_name_or_path'   , default=UpperCAmelCase__   , type=UpperCAmelCase__   , required=UpperCAmelCase__   , help='Path to pretrained model or model identifier from huggingface.co/models'   , )
										parser.add_argument(
										    '--config_name'   , default=''   , type=UpperCAmelCase__   , help='Pretrained config name or path if not the same as model_name'     )
										parser.add_argument(
										    '--tokenizer_name'   , default=UpperCAmelCase__   , type=UpperCAmelCase__   , help='Pretrained tokenizer name or path if not the same as model_name'   , )
										parser.add_argument(
										    '--cache_dir'   , default=''   , type=UpperCAmelCase__   , help='Where do you want to store the pre-trained models downloaded from huggingface.co'   , )
										parser.add_argument(
										    '--encoder_layerdrop'   , type=UpperCAmelCase__   , help='Encoder layer dropout probability (Optional). Goes into model.config'   , )
										parser.add_argument(
										    '--decoder_layerdrop'   , type=UpperCAmelCase__   , help='Decoder layer dropout probability (Optional). Goes into model.config'   , )
										parser.add_argument(
										    '--dropout'   , type=UpperCAmelCase__   , help='Dropout probability (Optional). Goes into model.config'   , )
										parser.add_argument(
										    '--attention_dropout'   , type=UpperCAmelCase__   , help='Attention dropout probability (Optional). Goes into model.config'   , )
										parser.add_argument('--learning_rate'   , default=5e-5   , type=UpperCAmelCase__   , help='The initial learning rate for Adam.'     )
										parser.add_argument(
										    '--lr_scheduler'   , default='linear'   , choices=UpperCAmelCase__   , metavar=UpperCAmelCase__   , type=UpperCAmelCase__   , help='Learning rate scheduler'   , )
										parser.add_argument('--weight_decay'   , default=0.0   , type=UpperCAmelCase__   , help='Weight decay if we apply some.'     )
										parser.add_argument('--adam_epsilon'   , default=1e-8   , type=UpperCAmelCase__   , help='Epsilon for Adam optimizer.'     )
										parser.add_argument('--warmup_steps'   , default=0   , type=UpperCAmelCase__   , help='Linear warmup over warmup_steps.'     )
										parser.add_argument('--num_workers'   , default=4   , type=UpperCAmelCase__   , help='kwarg passed to DataLoader'     )
										parser.add_argument('--num_train_epochs'   , dest='max_epochs'   , default=3   , type=UpperCAmelCase__     )
										parser.add_argument('--train_batch_size'   , default=32   , type=UpperCAmelCase__     )
										parser.add_argument('--eval_batch_size'   , default=32   , type=UpperCAmelCase__     )
										parser.add_argument('--adafactor'   , action='store_true'     )



class    _snake_case   (				pl.Callback					):

					"""simple docstring"""





					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__   , UpperCAmelCase__     )      ->	int:
										a_									=   trainer.lr_schedulers[0]['scheduler']
										a_									=   {F'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()     )}
										pl_module.logger.log_metrics(UpperCAmelCase__     )




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__   , UpperCAmelCase__     )      ->	int:
										rank_zero_info('***** Validation results *****'     )
										a_									=   trainer.callback_metrics
										# Log results
										for key in sorted(UpperCAmelCase__     ):
															if key not in ["log", "progress_bar"]:
																				rank_zero_info('{} = {}\n'.format(UpperCAmelCase__   , str(metrics[key]     )     )     )




					def  __SCREAMING_SNAKE_CASE (     self   , UpperCAmelCase__   , UpperCAmelCase__     )      ->	List[Any]:
										rank_zero_info('***** Test results *****'     )
										a_									=   trainer.callback_metrics
										# Log and save results to file
										a_									=   os.path.join(pl_module.hparams.output_dir   , 'test_results.txt'     )
										with open(UpperCAmelCase__   , 'w'     ) as writer:
															for key in sorted(UpperCAmelCase__     ):
																				if key not in ["log", "progress_bar"]:
																									rank_zero_info('{} = {}\n'.format(UpperCAmelCase__   , str(metrics[key]     )     )     )
																									writer.write('{} = {}\n'.format(UpperCAmelCase__   , str(metrics[key]     )     )     )



def 							a	(   _UpperCAmelCase					,  _UpperCAmelCase )				->				None:






					"""simple docstring"""







					parser.add_argument(
					    '--output_dir'					,  default=_UpperCAmelCase					,  type=_UpperCAmelCase					,  required=_UpperCAmelCase					,  help='The output directory where the model predictions and checkpoints will be written.'					,  )
					parser.add_argument(
					    '--fp16'					,  action='store_true'					,  help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'					,  )

					parser.add_argument(
					    '--fp16_opt_level'					,  type=_UpperCAmelCase					,  default='O2'					,  help=(
					        'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
					        'See details at https://nvidia.github.io/apex/amp.html'
					    )					,  )
					parser.add_argument('--n_tpu_cores'					,  dest='tpu_cores'					,  type=_UpperCAmelCase )
					parser.add_argument('--max_grad_norm'					,  dest='gradient_clip_val'					,  default=1.0					,  type=_UpperCAmelCase					,  help='Max gradient norm' )
					parser.add_argument('--do_train'					,  action='store_true'					,  help='Whether to run training.' )
					parser.add_argument('--do_predict'					,  action='store_true'					,  help='Whether to run predictions on the test set.' )
					parser.add_argument(
					    '--gradient_accumulation_steps'					,  dest='accumulate_grad_batches'					,  type=_UpperCAmelCase					,  default=1					,  help='Number of updates steps to accumulate before performing a backward/update pass.'					,  )
					parser.add_argument('--seed'					,  type=_UpperCAmelCase					,  default=4_2					,  help='random seed for initialization' )
					parser.add_argument(
					    '--data_dir'					,  default=_UpperCAmelCase					,  type=_UpperCAmelCase					,  required=_UpperCAmelCase					,  help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'					,  )





def 							a	(   _UpperCAmelCase					,  _UpperCAmelCase					,  _UpperCAmelCase=None					,  _UpperCAmelCase=True					,  _UpperCAmelCase=[]					,  _UpperCAmelCase=None					,  _UpperCAmelCase=None					,  **_UpperCAmelCase					,  )				->				Dict:






					"""simple docstring"""







					pl.seed_everything(args.seed )

					# init model
					a_									=   Path(model.hparams.output_dir )
					odir.mkdir(exist_ok=_UpperCAmelCase )

					# add custom checkpoints
					if checkpoint_callback is None:
										a_									=   pl.callbacks.ModelCheckpoint(
										    filepath=args.output_dir					,  prefix='checkpoint'					,  monitor='val_loss'					,  mode='min'					,  save_top_k=1 )
					if early_stopping_callback:
										extra_callbacks.append(_UpperCAmelCase )
					if logging_callback is None:
										a_									=   LoggingCallback()

					a_									=   {}

					# TODO: remove with PyTorch 1.6 since pl uses native amp
					if args.fpaa:
										a_									=   1_6
										a_									=   args.fpaa_opt_level

					if args.gpus > 1:
										a_									=   'ddp'

					a_									=   args.accumulate_grad_batches
					a_									=   extra_train_kwargs.get('accelerator'					,  _UpperCAmelCase )
					a_									=   extra_train_kwargs.get('profiler'					,  _UpperCAmelCase )

					a_									=   pl.Trainer.from_argparse_args(
					    _UpperCAmelCase					,  weights_summary=_UpperCAmelCase					,  callbacks=[logging_callback] + extra_callbacks					,  logger=_UpperCAmelCase					,  checkpoint_callback=_UpperCAmelCase					,  **_UpperCAmelCase					,  )

					if args.do_train:
										trainer.fit(_UpperCAmelCase )

					return trainer









'''simple docstring'''
import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm

def 							a	(   _UpperCAmelCase					,  _UpperCAmelCase = False )				->				List[str]:






					"""simple docstring"""







					if device_specific:
										seed += AcceleratorState().process_index
					random.seed(_UpperCAmelCase )
					np.random.seed(_UpperCAmelCase )
					torch.manual_seed(_UpperCAmelCase )
					if is_xpu_available():
										torch.xpu.manual_seed_all(_UpperCAmelCase )
					elif is_npu_available():
										torch.npu.manual_seed_all(_UpperCAmelCase )
					else:
										torch.cuda.manual_seed_all(_UpperCAmelCase )
					# ^^ safe to call this function even if cuda is not available
					if is_tpu_available():
										xm.set_rng_state(_UpperCAmelCase )

def 							a	(   _UpperCAmelCase = None					,  _UpperCAmelCase = None )				->				Tuple:






					"""simple docstring"""







					if rng_type == RNGType.TORCH:
										a_									=   torch.get_rng_state()
					elif rng_type == RNGType.CUDA:
										a_									=   torch.cuda.get_rng_state()
					elif rng_type == RNGType.XLA:
										assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
										a_									=   torch.tensor(xm.get_rng_state() )
					elif rng_type == RNGType.NPU:
										assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
										a_									=   torch.npu.get_rng_state()
					elif rng_type == RNGType.XPU:
										assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
										a_									=   torch.xpu.get_rng_state()
					elif rng_type == RNGType.GENERATOR:
										assert generator is not None, "Need a generator to synchronize its seed."
										a_									=   generator.get_state()

					# Broadcast the rng state from device 0 to other devices
					a_									=   AcceleratorState()
					if state.distributed_type == DistributedType.TPU:
										a_									=   rng_state.to(xm.xla_device() )
										xm.collective_broadcast([rng_state] )
										xm.mark_step()
										a_									=   rng_state.cpu()
					elif (
					    state.distributed_type in CUDA_DISTRIBUTED_TYPES
					    or state.distributed_type == DistributedType.MULTI_NPU
					    or state.distributed_type == DistributedType.MULTI_XPU
					):
										a_									=   rng_state.to(state.device )
										torch.distributed.broadcast(_UpperCAmelCase					,  0 )
										a_									=   rng_state.cpu()
					elif state.distributed_type == DistributedType.MULTI_CPU:
										torch.distributed.broadcast(_UpperCAmelCase					,  0 )

					# Set the broadcast rng state
					if rng_type == RNGType.TORCH:
										torch.set_rng_state(_UpperCAmelCase )
					elif rng_type == RNGType.CUDA:
										torch.cuda.set_rng_state(_UpperCAmelCase )
					elif rng_type == RNGType.NPU:
										torch.npu.set_rng_state(_UpperCAmelCase )
					elif rng_type == RNGType.XPU:
										torch.xpu.set_rng_state(_UpperCAmelCase )
					elif rng_type == RNGType.XLA:
										xm.set_rng_state(rng_state.item() )
					elif rng_type == RNGType.GENERATOR:
										generator.set_state(_UpperCAmelCase )





def 							a	(   _UpperCAmelCase					,  _UpperCAmelCase = None )				->				Any:






					"""simple docstring"""







					for rng_type in rng_types:
										synchronize_rng_state(RNGType(_UpperCAmelCase )					,  generator=_UpperCAmelCase )

