






import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


UpperCamelCase__						:      Optional[Any]								=					logging.getLogger(__name__)

require_version("""pytorch_lightning>=1.0.4""")

UpperCamelCase__						:      Tuple								=					{
    """base""": AutoModel,
    """sequence-classification""": AutoModelForSequenceClassification,
    """question-answering""": AutoModelForQuestionAnswering,
    """pretraining""": AutoModelForPreTraining,
    """token-classification""": AutoModelForTokenClassification,
    """language-modeling""": AutoModelWithLMHead,
    """summarization""": AutoModelForSeqaSeqLM,
    """translation""": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
UpperCamelCase__						:      Union[str, Any]								=					{
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
UpperCamelCase__						:      Tuple								=					sorted(arg_to_scheduler.keys())
UpperCamelCase__						:      Optional[int]								=					"""{""" + """, """.join(arg_to_scheduler_choices) + """}"""







class 				lowerCamelCase_     (	pl.LightningModule     ):
       def __init__(						self		:					Tuple							,__lowerCamelCase		:					Tuple							,__lowerCamelCase		:					int=None							,__lowerCamelCase		:					Any="base"							,__lowerCamelCase		:					Dict=None							,__lowerCamelCase		:					List[Any]=None							,__lowerCamelCase		:					Union[str, Any]=None							,**__lowerCamelCase		:					Union[str, Any]							,):





        '''simple docstring'''



        super().__init__()
        # TODO: move to self.save_hyperparameters()
        # self.save_hyperparameters()
        # can also expand arguments into trainer signature for easier reading

        self.save_hyperparameters(a_					)
        a		=			0
        a		=			Path(self.hparams.output_dir					)
        a		=			self.hparams.cache_dir if self.hparams.cache_dir else None
        if config is None:
         a		=			AutoConfig.from_pretrained(
             self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path							,**({'''num_labels''': num_labels} if num_labels is not None else {})							,cache_dir=a_							,**a_							,)
        else:
         a		=			config

        a		=			('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
        for p in extra_model_params:
         if getattr(self.hparams							,a_							,a_					):
          assert hasattr(self.config							,a_					), F"""model config doesn't have a `{p}` attribute"""
          setattr(self.config							,a_							,getattr(self.hparams							,a_					)					)

        if tokenizer is None:
         a		=			AutoTokenizer.from_pretrained(
             self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path							,cache_dir=a_							,)
        else:
         a		=			tokenizer
        a		=			MODEL_MODES[mode]
        if model is None:
         a		=			self.model_type.from_pretrained(
             self.hparams.model_name_or_path							,from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path					)							,config=self.config							,cache_dir=a_							,)
        else:
         a		=			model
       def  SCREAMING_SNAKE_CASE_ (						self		:					int							,*__lowerCamelCase		:					Union[str, Any]							,**__lowerCamelCase		:					Dict					):





        '''simple docstring'''



        a		=			self.model_type.from_pretrained(*a_							,**a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					int					):





        '''simple docstring'''



        a		=			arg_to_scheduler[self.hparams.lr_scheduler]
        a		=			get_schedule_func(
            self.opt							,num_warmup_steps=self.hparams.warmup_steps							,num_training_steps=self.total_steps()					)
        a		=			{'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
        return scheduler
       def  SCREAMING_SNAKE_CASE_ (						self		:					Optional[int]					):





        '''simple docstring'''



        a		=			self.model
        a		=			['''bias''', '''LayerNorm.weight''']
        a		=			[
            {
                '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay					)],
                '''weight_decay''': self.hparams.weight_decay,
            },
            {
                '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay					)],
                '''weight_decay''': 0.0,
            },
        ]
        if self.hparams.adafactor:
         a		=			Adafactor(
             a_							,lr=self.hparams.learning_rate							,scale_parameter=a_							,relative_step=a_					)

        else:
         a		=			AdamW(
             a_							,lr=self.hparams.learning_rate							,eps=self.hparams.adam_epsilon					)
        a		=			optimizer

        a		=			self.get_lr_scheduler()

        return [optimizer], [scheduler]
       def  SCREAMING_SNAKE_CASE_ (						self		:					str							,__lowerCamelCase		:					Optional[Any]							,__lowerCamelCase		:					List[str]					):





        '''simple docstring'''



        return self.validation_step(a_							,a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]							,__lowerCamelCase		:					List[str]					):





        '''simple docstring'''



        return self.validation_end(a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					str					):





        '''simple docstring'''



        a		=			max(1							,self.hparams.gpus					)  # TODO: consider num_tpu_cores
        a		=			self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
        return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
       def  SCREAMING_SNAKE_CASE_ (						self		:					List[str]							,__lowerCamelCase		:					Dict					):





        '''simple docstring'''



        if mode == "test":
         a		=			len(self.test_dataloader().dataset					)
        else:
         a		=			self.get_dataloader('''train'''							,self.hparams.train_batch_size							,shuffle=a_					)
         a		=			len(self.train_dataloader().dataset					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					Any							,__lowerCamelCase		:					List[Any]							,__lowerCamelCase		:					Optional[Any]							,__lowerCamelCase		:					Union[str, Any] = False					):





        '''simple docstring'''



        raise NotImplementedError('''You must implement this for your task'''					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					int					):





        '''simple docstring'''



        return self.train_loader
       def  SCREAMING_SNAKE_CASE_ (						self		:					Any					):





        '''simple docstring'''



        return self.get_dataloader('''dev'''							,self.hparams.eval_batch_size							,shuffle=a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					Optional[Any]					):





        '''simple docstring'''



        return self.get_dataloader('''test'''							,self.hparams.eval_batch_size							,shuffle=a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					List[str]							,__lowerCamelCase		:					str					):





        '''simple docstring'''



        return os.path.join(
            self.hparams.data_dir							,'''cached_{}_{}_{}'''.format(
                a_							,list(filter(a_							,self.hparams.model_name_or_path.split('''/'''					)					)					).pop()							,str(self.hparams.max_seq_length					)							,)							,)
       @pl.utilities.rank_zero_only
       def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]							,__lowerCamelCase		:					Optional[Any]					):





        '''simple docstring'''



        a		=			self.output_dir.joinpath('''best_tfmr'''					)
        a		=			self.step_count
        self.model.save_pretrained(a_					)
        self.tokenizer.save_pretrained(a_					)




       @staticmethod
       def  SCREAMING_SNAKE_CASE_ (						__lowerCamelCase		:					Optional[int]							,__lowerCamelCase		:					int					):





        '''simple docstring'''



        parser.add_argument(
            '''--model_name_or_path'''							,default=a_							,type=a_							,required=a_							,help='''Path to pretrained model or model identifier from huggingface.co/models'''							,)
        parser.add_argument(
            '''--config_name'''							,default=''''''							,type=a_							,help='''Pretrained config name or path if not the same as model_name'''					)
        parser.add_argument(
            '''--tokenizer_name'''							,default=a_							,type=a_							,help='''Pretrained tokenizer name or path if not the same as model_name'''							,)
        parser.add_argument(
            '''--cache_dir'''							,default=''''''							,type=a_							,help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''							,)
        parser.add_argument(
            '''--encoder_layerdrop'''							,type=a_							,help='''Encoder layer dropout probability (Optional). Goes into model.config'''							,)
        parser.add_argument(
            '''--decoder_layerdrop'''							,type=a_							,help='''Decoder layer dropout probability (Optional). Goes into model.config'''							,)
        parser.add_argument(
            '''--dropout'''							,type=a_							,help='''Dropout probability (Optional). Goes into model.config'''							,)
        parser.add_argument(
            '''--attention_dropout'''							,type=a_							,help='''Attention dropout probability (Optional). Goes into model.config'''							,)
        parser.add_argument('''--learning_rate'''							,default=5e-5							,type=a_							,help='''The initial learning rate for Adam.'''					)
        parser.add_argument(
            '''--lr_scheduler'''							,default='''linear'''							,choices=a_							,metavar=a_							,type=a_							,help='''Learning rate scheduler'''							,)
        parser.add_argument('''--weight_decay'''							,default=0.0							,type=a_							,help='''Weight decay if we apply some.'''					)
        parser.add_argument('''--adam_epsilon'''							,default=1e-8							,type=a_							,help='''Epsilon for Adam optimizer.'''					)
        parser.add_argument('''--warmup_steps'''							,default=0							,type=a_							,help='''Linear warmup over warmup_steps.'''					)
        parser.add_argument('''--num_workers'''							,default=4							,type=a_							,help='''kwarg passed to DataLoader'''					)
        parser.add_argument('''--num_train_epochs'''							,dest='''max_epochs'''							,default=3							,type=a_					)
        parser.add_argument('''--train_batch_size'''							,default=32							,type=a_					)
        parser.add_argument('''--eval_batch_size'''							,default=32							,type=a_					)
        parser.add_argument('''--adafactor'''							,action='''store_true'''					)







class 				lowerCamelCase_     (	pl.Callback     ):
       def  SCREAMING_SNAKE_CASE_ (						self		:					Union[str, Any]							,__lowerCamelCase		:					str							,__lowerCamelCase		:					List[Any]					):





        '''simple docstring'''



        a		=			trainer.lr_schedulers[0]['''scheduler''']
        a		=			{F"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()					)}
        pl_module.logger.log_metrics(a_					)
       def  SCREAMING_SNAKE_CASE_ (						self		:					int							,__lowerCamelCase		:					List[Any]							,__lowerCamelCase		:					Optional[Any]					):





        '''simple docstring'''



        rank_zero_info('''***** Validation results *****'''					)
        a		=			trainer.callback_metrics
        # Log results
        for key in sorted(a_					):
         if key not in ["log", "progress_bar"]:
          rank_zero_info('''{} = {}\n'''.format(a_							,str(metrics[key]					)					)					)




       def  SCREAMING_SNAKE_CASE_ (						self		:					Tuple							,__lowerCamelCase		:					Tuple							,__lowerCamelCase		:					Any					):





        '''simple docstring'''



        rank_zero_info('''***** Test results *****'''					)
        a		=			trainer.callback_metrics
        # Log and save results to file
        a		=			os.path.join(pl_module.hparams.output_dir							,'''test_results.txt'''					)
        with open(a_							,'''w'''					) as writer:
         for key in sorted(a_					):
          if key not in ["log", "progress_bar"]:
           rank_zero_info('''{} = {}\n'''.format(a_							,str(metrics[key]					)					)					)
           writer.write('''{} = {}\n'''.format(a_							,str(metrics[key]					)					)					)





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_  )   ->  Optional[int]:


 """simple docstring"""



 parser.add_argument(
     '''--output_dir''',  default=UpperCamelCase__,  type=UpperCamelCase__,  required=UpperCamelCase__,  help='''The output directory where the model predictions and checkpoints will be written.''',  )
 parser.add_argument(
     '''--fp16''',  action='''store_true''',  help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit''',  )

 parser.add_argument(
     '''--fp16_opt_level''',  type=UpperCamelCase__,  default='''O2''',  help=(
         '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
         '''See details at https://nvidia.github.io/apex/amp.html'''
     ),  )
 parser.add_argument('''--n_tpu_cores''',  dest='''tpu_cores''',  type=UpperCamelCase__  )
 parser.add_argument('''--max_grad_norm''',  dest='''gradient_clip_val''',  default=1.0,  type=UpperCamelCase__,  help='''Max gradient norm'''  )
 parser.add_argument('''--do_train''',  action='''store_true''',  help='''Whether to run training.'''  )
 parser.add_argument('''--do_predict''',  action='''store_true''',  help='''Whether to run predictions on the test set.'''  )
 parser.add_argument(
     '''--gradient_accumulation_steps''',  dest='''accumulate_grad_batches''',  type=UpperCamelCase__,  default=1,  help='''Number of updates steps to accumulate before performing a backward/update pass.''',  )
 parser.add_argument('''--seed''',  type=UpperCamelCase__,  default=4_2,  help='''random seed for initialization'''  )
 parser.add_argument(
     '''--data_dir''',  default=UpperCamelCase__,  type=UpperCamelCase__,  required=UpperCamelCase__,  help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.''',  )





def   SCREAMING_SNAKE_CASE__						(   snake_case_,  snake_case_,  snake_case_=None,  snake_case_=True,  snake_case_=[],  snake_case_=None,  snake_case_=None,  **snake_case_,  )   ->  Optional[int]:


 """simple docstring"""



 pl.seed_everything(args.seed  )

 # init model
 a		=			Path(model.hparams.output_dir  )
 odir.mkdir(exist_ok=UpperCamelCase__  )

 # add custom checkpoints
 if checkpoint_callback is None:
  a		=			pl.callbacks.ModelCheckpoint(
      filepath=args.output_dir,  prefix='''checkpoint''',  monitor='''val_loss''',  mode='''min''',  save_top_k=1  )
 if early_stopping_callback:
  extra_callbacks.append(UpperCamelCase__  )
 if logging_callback is None:
  a		=			LoggingCallback()

 a		=			{}

 # TODO: remove with PyTorch 1.6 since pl uses native amp
 if args.fpaa:
  a		=			1_6
  a		=			args.fpaa_opt_level

 if args.gpus > 1:
  a		=			'''ddp'''

 a		=			args.accumulate_grad_batches
 a		=			extra_train_kwargs.get('''accelerator''',  UpperCamelCase__  )
 a		=			extra_train_kwargs.get('''profiler''',  UpperCamelCase__  )

 a		=			pl.Trainer.from_argparse_args(
     UpperCamelCase__,  weights_summary=UpperCamelCase__,  callbacks=[logging_callback] + extra_callbacks,  logger=UpperCamelCase__,  checkpoint_callback=UpperCamelCase__,  **UpperCamelCase__,  )

 if args.do_train:
  trainer.fit(UpperCamelCase__  )

 return trainer













def   SCREAMING_SNAKE_CASE__						(   snake_case_  )   ->  int:


	"""simple docstring"""



	return n * (2 * n - 1)





def   SCREAMING_SNAKE_CASE__						(   snake_case_  )   ->  bool:


	"""simple docstring"""



	a		=			(1 + 2_4 * n) ** 0.5
	return ((1 + root) / 6) % 1 == 0





def   SCREAMING_SNAKE_CASE__						(   snake_case_ = 1_4_4  )   ->  int:


	"""simple docstring"""



	a		=			start
	a		=			hexagonal_num(snake_case_  )
	while not is_pentagonal(snake_case_  ):
		n += 1
		a		=			hexagonal_num(snake_case_  )
	return num


if __name__ == "__main__":
				print(F"{solution()} = ")





