
'''simple docstring'''







import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


_lowercase				:						Optional[int]												= logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

_lowercase				:						Any												= {
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
_lowercase				:						List[Any]												= {
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
_lowercase				:						Any												= sorted(arg_to_scheduler.keys())
_lowercase				:						List[str]												= "{" + ", ".join(arg_to_scheduler_choices) + "}"






class 				lowerCAmelCase__     (						pl.LightningModule  ):






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE="base"       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											super().__init__()
											# TODO: move to self.save_hyperparameters()
											# self.save_hyperparameters()
											# can also expand arguments into trainer signature for easier reading

											self.save_hyperparameters(__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Union[str, Any]									= 0
											lowercase_			:  int									= Path(self.hparams.output_dir	)
											lowercase_			:  Optional[Any]									= self.hparams.cache_dir if self.hparams.cache_dir else None
											if config is None:
																	lowercase_			:  Optional[int]									= AutoConfig.from_pretrained(
																	    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path       ,							**({'''num_labels''': num_labels} if num_labels is not None else {})       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)
											else:
																	lowercase_			:  PretrainedConfig									= config

											lowercase_			:  Any									= ('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
											for p in extra_model_params:
																	if getattr(self.hparams       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):
																							assert hasattr(self.config       ,							__SCREAMING_SNAKE_CASE	), F'''model config doesn\'t have a `{p}` attribute'''
																							setattr(self.config       ,							__SCREAMING_SNAKE_CASE       ,							getattr(self.hparams       ,							__SCREAMING_SNAKE_CASE	)	)

											if tokenizer is None:
																	lowercase_			:  Optional[Any]									= AutoTokenizer.from_pretrained(
																	    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							)
											else:
																	lowercase_			:  PreTrainedTokenizer									= tokenizer
											lowercase_			:  Any									= MODEL_MODES[mode]
											if model is None:
																	lowercase_			:  List[Any]									= self.model_type.from_pretrained(
																	    self.hparams.model_name_or_path       ,							from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path	)       ,							config=self.config       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							)
											else:
																	lowercase_			:  Dict									= model






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= self.model_type.from_pretrained(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= arg_to_scheduler[self.hparams.lr_scheduler]
											lowercase_			:  int									= get_schedule_func(
											    self.opt       ,							num_warmup_steps=self.hparams.warmup_steps       ,							num_training_steps=self.total_steps()	)
											lowercase_			:  Any									= {'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
											return scheduler






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= self.model
											lowercase_			:  Dict									= ['''bias''', '''LayerNorm.weight''']
											lowercase_			:  List[Any]									= [
											    {
											        '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay	)],
											        '''weight_decay''': self.hparams.weight_decay,
											    },
											    {
											        '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay	)],
											        '''weight_decay''': 0.0,
											    },
											]
											if self.hparams.adafactor:
																	lowercase_			:  Optional[int]									= Adafactor(
																	    __SCREAMING_SNAKE_CASE       ,							lr=self.hparams.learning_rate       ,							scale_parameter=__SCREAMING_SNAKE_CASE       ,							relative_step=__SCREAMING_SNAKE_CASE	)

											else:
																	lowercase_			:  Union[str, Any]									= AdamW(
																	    __SCREAMING_SNAKE_CASE       ,							lr=self.hparams.learning_rate       ,							eps=self.hparams.adam_epsilon	)
											lowercase_			:  Tuple									= optimizer

											lowercase_			:  Tuple									= self.get_lr_scheduler()

											return [optimizer], [scheduler]






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return self.validation_step(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return self.validation_end(__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Any									= max(1       ,							self.hparams.gpus	)  # TODO: consider num_tpu_cores
											lowercase_			:  Any									= self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
											return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											if mode == "test":
																	lowercase_			:  Optional[Any]									= len(self.test_dataloader().dataset	)
											else:
																	lowercase_			:  Any									= self.get_dataloader('''train'''       ,							self.hparams.train_batch_size       ,							shuffle=__SCREAMING_SNAKE_CASE	)
																	lowercase_			:  Tuple									= len(self.train_dataloader().dataset	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = False	):


											"""simple docstring"""





											raise NotImplementedError('''You must implement this for your task'''	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											return self.train_loader






					def 			_snake_case					(     self	):


											"""simple docstring"""





											return self.get_dataloader('''dev'''       ,							self.hparams.eval_batch_size       ,							shuffle=__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											return self.get_dataloader('''test'''       ,							self.hparams.eval_batch_size       ,							shuffle=__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return os.path.join(
											    self.hparams.data_dir       ,							'''cached_{}_{}_{}'''.format(
											        __SCREAMING_SNAKE_CASE       ,							list(filter(__SCREAMING_SNAKE_CASE       ,							self.hparams.model_name_or_path.split('''/'''	)	)	).pop()       ,							str(self.hparams.max_seq_length	)       ,							)       ,							)






					@pl.utilities.rank_zero_only
					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= self.output_dir.joinpath('''best_tfmr'''	)
											lowercase_			:  str									= self.step_count
											self.model.save_pretrained(__SCREAMING_SNAKE_CASE	)
											self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE	)






					@staticmethod
					def 			_snake_case					(     __SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											parser.add_argument(
											    '''--model_name_or_path'''       ,							default=__SCREAMING_SNAKE_CASE       ,							type=__SCREAMING_SNAKE_CASE       ,							required=__SCREAMING_SNAKE_CASE       ,							help='''Path to pretrained model or model identifier from huggingface.co/models'''       ,							)
											parser.add_argument(
											    '''--config_name'''       ,							default=''''''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Pretrained config name or path if not the same as model_name'''	)
											parser.add_argument(
											    '''--tokenizer_name'''       ,							default=__SCREAMING_SNAKE_CASE       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Pretrained tokenizer name or path if not the same as model_name'''       ,							)
											parser.add_argument(
											    '''--cache_dir'''       ,							default=''''''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''       ,							)
											parser.add_argument(
											    '''--encoder_layerdrop'''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Encoder layer dropout probability (Optional). Goes into model.config'''       ,							)
											parser.add_argument(
											    '''--decoder_layerdrop'''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Decoder layer dropout probability (Optional). Goes into model.config'''       ,							)
											parser.add_argument(
											    '''--dropout'''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Dropout probability (Optional). Goes into model.config'''       ,							)
											parser.add_argument(
											    '''--attention_dropout'''       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Attention dropout probability (Optional). Goes into model.config'''       ,							)
											parser.add_argument('''--learning_rate'''       ,							default=5E-5       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''The initial learning rate for Adam.'''	)
											parser.add_argument(
											    '''--lr_scheduler'''       ,							default='''linear'''       ,							choices=__SCREAMING_SNAKE_CASE       ,							metavar=__SCREAMING_SNAKE_CASE       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Learning rate scheduler'''       ,							)
											parser.add_argument('''--weight_decay'''       ,							default=0.0       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Weight decay if we apply some.'''	)
											parser.add_argument('''--adam_epsilon'''       ,							default=1E-8       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Epsilon for Adam optimizer.'''	)
											parser.add_argument('''--warmup_steps'''       ,							default=0       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''Linear warmup over warmup_steps.'''	)
											parser.add_argument('''--num_workers'''       ,							default=4       ,							type=__SCREAMING_SNAKE_CASE       ,							help='''kwarg passed to DataLoader'''	)
											parser.add_argument('''--num_train_epochs'''       ,							dest='''max_epochs'''       ,							default=3       ,							type=__SCREAMING_SNAKE_CASE	)
											parser.add_argument('''--train_batch_size'''       ,							default=32       ,							type=__SCREAMING_SNAKE_CASE	)
											parser.add_argument('''--eval_batch_size'''       ,							default=32       ,							type=__SCREAMING_SNAKE_CASE	)
											parser.add_argument('''--adafactor'''       ,							action='''store_true'''	)







class 				lowerCAmelCase__     (						pl.Callback  ):






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  str									= trainer.lr_schedulers[0]['''scheduler''']
											lowercase_			:  Dict									= {F'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()	)}
											pl_module.logger.log_metrics(__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											rank_zero_info('''***** Validation results *****'''	)
											lowercase_			:  List[str]									= trainer.callback_metrics
											# Log results
											for key in sorted(__SCREAMING_SNAKE_CASE	):
																	if key not in ["log", "progress_bar"]:
																							rank_zero_info('''{} = {}\n'''.format(__SCREAMING_SNAKE_CASE       ,							str(metrics[key]	)	)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											rank_zero_info('''***** Test results *****'''	)
											lowercase_			:  Dict									= trainer.callback_metrics
											# Log and save results to file
											lowercase_			:  Tuple									= os.path.join(pl_module.hparams.output_dir       ,							'''test_results.txt'''	)
											with open(__SCREAMING_SNAKE_CASE       ,							'''w'''	) as writer:
																	for key in sorted(__SCREAMING_SNAKE_CASE	):
																							if key not in ["log", "progress_bar"]:
																													rank_zero_info('''{} = {}\n'''.format(__SCREAMING_SNAKE_CASE       ,							str(metrics[key]	)	)	)
																													writer.write('''{} = {}\n'''.format(__SCREAMING_SNAKE_CASE       ,							str(metrics[key]	)	)	)







def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				str   ,   __SCREAMING_SNAKE_CASE      :				str							):



						"""simple docstring"""





						parser.add_argument(
						    '''--output_dir'''   ,   default=__SCREAMING_SNAKE_CASE   ,   type=__SCREAMING_SNAKE_CASE   ,   required=__SCREAMING_SNAKE_CASE   ,   help='''The output directory where the model predictions and checkpoints will be written.'''   ,   )
						parser.add_argument(
						    '''--fp16'''   ,   action='''store_true'''   ,   help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'''   ,   )

						parser.add_argument(
						    '''--fp16_opt_level'''   ,   type=__SCREAMING_SNAKE_CASE   ,   default='''O2'''   ,   help=(
						        '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
						        '''See details at https://nvidia.github.io/apex/amp.html'''
						    )   ,   )
						parser.add_argument('''--n_tpu_cores'''   ,   dest='''tpu_cores'''   ,   type=__SCREAMING_SNAKE_CASE							)
						parser.add_argument('''--max_grad_norm'''   ,   dest='''gradient_clip_val'''   ,   default=1.0   ,   type=__SCREAMING_SNAKE_CASE   ,   help='''Max gradient norm'''							)
						parser.add_argument('''--do_train'''   ,   action='''store_true'''   ,   help='''Whether to run training.'''							)
						parser.add_argument('''--do_predict'''   ,   action='''store_true'''   ,   help='''Whether to run predictions on the test set.'''							)
						parser.add_argument(
						    '''--gradient_accumulation_steps'''   ,   dest='''accumulate_grad_batches'''   ,   type=__SCREAMING_SNAKE_CASE   ,   default=1   ,   help='''Number of updates steps to accumulate before performing a backward/update pass.'''   ,   )
						parser.add_argument('''--seed'''   ,   type=__SCREAMING_SNAKE_CASE   ,   default=42   ,   help='''random seed for initialization'''							)
						parser.add_argument(
						    '''--data_dir'''   ,   default=__SCREAMING_SNAKE_CASE   ,   type=__SCREAMING_SNAKE_CASE   ,   required=__SCREAMING_SNAKE_CASE   ,   help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.'''   ,   )




def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				BaseTransformer   ,   __SCREAMING_SNAKE_CASE      :				argparse.Namespace   ,   __SCREAMING_SNAKE_CASE      :				List[Any]=None   ,   __SCREAMING_SNAKE_CASE      :				Optional[Any]=True   ,   __SCREAMING_SNAKE_CASE      :				Union[str, Any]=[]   ,   __SCREAMING_SNAKE_CASE      :				int=None   ,   __SCREAMING_SNAKE_CASE      :				Any=None   ,   **__SCREAMING_SNAKE_CASE      :				Union[str, Any]   ,   ):



						"""simple docstring"""





						pl.seed_everything(args.seed							)

						# init model
						lowercase_			:  Optional[int]									= Path(model.hparams.output_dir							)
						odir.mkdir(exist_ok=__SCREAMING_SNAKE_CASE							)

						# add custom checkpoints
						if checkpoint_callback is None:
												lowercase_			:  Optional[Any]									= pl.callbacks.ModelCheckpoint(
												    filepath=args.output_dir   ,   prefix='''checkpoint'''   ,   monitor='''val_loss'''   ,   mode='''min'''   ,   save_top_k=1							)
						if early_stopping_callback:
												extra_callbacks.append(__SCREAMING_SNAKE_CASE							)
						if logging_callback is None:
												lowercase_			:  str									= LoggingCallback()

						lowercase_			:  Optional[int]									= {}

						# TODO: remove with PyTorch 1.6 since pl uses native amp
						if args.fpaa:
												lowercase_			:  str									= 16
												lowercase_			:  Optional[int]									= args.fpaa_opt_level

						if args.gpus > 1:
												lowercase_			:  int									= '''ddp'''

						lowercase_			:  Optional[int]									= args.accumulate_grad_batches
						lowercase_			:  List[Any]									= extra_train_kwargs.get('''accelerator'''   ,   __SCREAMING_SNAKE_CASE							)
						lowercase_			:  Union[str, Any]									= extra_train_kwargs.get('''profiler'''   ,   __SCREAMING_SNAKE_CASE							)

						lowercase_			:  Optional[Any]									= pl.Trainer.from_argparse_args(
						    __SCREAMING_SNAKE_CASE   ,   weights_summary=__SCREAMING_SNAKE_CASE   ,   callbacks=[logging_callback] + extra_callbacks   ,   logger=__SCREAMING_SNAKE_CASE   ,   checkpoint_callback=__SCREAMING_SNAKE_CASE   ,   **__SCREAMING_SNAKE_CASE   ,   )

						if args.do_train:
												trainer.fit(__SCREAMING_SNAKE_CASE							)

						return trainer





'''simple docstring'''







def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				int = 998001							):



						"""simple docstring"""





						lowercase_			:  Dict									= 0
						for i in range(999   ,   99   ,   -1							):  # 3 digit numbers range from 999 down to 100
												for j in range(999   ,   99   ,   -1							):
																		lowercase_			:  int									= str(i * j							)
																		if product_string == product_string[::-1] and i * j < n:
																								lowercase_			:  str									= max(__SCREAMING_SNAKE_CASE   ,   i * j							)
						return answer


if __name__ == "__main__":
							print(f"""{solution() = }""")



