




"""simple docstring"""

from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
						from .data.data_collator import DataCollator
						from .modeling_utils import PreTrainedModel
						from .tokenization_utils_base import PreTrainedTokenizerBase
						from .trainer_callback import TrainerCallback
						from .trainer_utils import EvalPrediction, PredictionOutput
						from .training_args import TrainingArguments


__lowerCamelCase								=						logging.get_logger(__name__)



class  _snake_case				(							A__       ):

				'''simple docstring'''




				def __init__(							self      :  List[Any]					,							snake_case      :  Union["PreTrainedModel", nn.Module] = None					,							snake_case      :  "TrainingArguments" = None					,							snake_case      :  Optional["DataCollator"] = None					,							snake_case      :  Optional[Dataset] = None					,							snake_case      :  Optional[Union[Dataset, Dict[str, Dataset]]] = None					,							snake_case      :  Optional["PreTrainedTokenizerBase"] = None					,							snake_case      :  Optional[Callable[[], "PreTrainedModel"]] = None					,							snake_case      :  Optional[Callable[["EvalPrediction"], Dict]] = None					,							snake_case      :  Optional[List["TrainerCallback"]] = None					,							snake_case      :  Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None)					,							snake_case      :  Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None					,							):
								super().__init__(
								    model=snake_case					,							args=snake_case					,							data_collator=snake_case					,							train_dataset=snake_case					,							eval_dataset=snake_case					,							tokenizer=snake_case					,							model_init=snake_case					,							compute_metrics=snake_case					,							callbacks=snake_case					,							optimizers=snake_case					,							preprocess_logits_for_metrics=snake_case					,							)

								# Override self.model.generation_config if a GenerationConfig is specified in args.
								# Priority: args.generation_config > model.generation_config > default GenerationConfig.
								if self.args.generation_config is not None:
												UpperCAmelCase_					:Optional[int]      			=      self.load_generation_config(self.args.generation_config     )
												UpperCAmelCase_					:Dict      			=      gen_config





				@staticmethod
				def 				snake_case_ (							snake_case      :  Union[str, GenerationConfig]     ):

								# GenerationConfig provided, nothing to do
								if isinstance(snake_case					,							snake_case     ):
												return deepcopy(snake_case     )

								# str or Path
								UpperCAmelCase_					:Optional[Any]      			=      Path(snake_case     ) if isinstance(snake_case					,							snake_case     ) else gen_config_arg
								UpperCAmelCase_					:List[Any]      			=      None

								# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
								# This step is required in order to determine config_file_name
								if pretrained_model_name.is_file():
												UpperCAmelCase_					:List[Any]      			=      pretrained_model_name.name
												UpperCAmelCase_					:Tuple      			=      pretrained_model_name.parent
								# dir path
								elif pretrained_model_name.is_dir():
												pass
								# model id or URL
								else:
												UpperCAmelCase_					:Dict      			=      gen_config_arg

								UpperCAmelCase_					:Optional[int]      			=      GenerationConfig.from_pretrained(snake_case					,							snake_case     )
								return gen_config





				def 				snake_case_ (							self      :  Tuple					,							snake_case      :  Optional[Dataset] = None					,							snake_case      :  Optional[List[str]] = None					,							snake_case      :  str = "eval"					,							**snake_case      :  Any					,							):
								UpperCAmelCase_					:Optional[int]      			=      gen_kwargs.copy()
								if gen_kwargs.get('''max_length'''     ) is None and gen_kwargs.get('''max_new_tokens'''     ) is None:
												UpperCAmelCase_					:str      			=      self.args.generation_max_length
								UpperCAmelCase_					:Dict      			=      (
								    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''     ) is not None else self.args.generation_num_beams
								)
								UpperCAmelCase_					:Tuple      			=      gen_kwargs

								return super().evaluate(snake_case					,							ignore_keys=snake_case					,							metric_key_prefix=snake_case     )





				def 				snake_case_ (							self      :  List[str]					,							snake_case      :  Dataset					,							snake_case      :  Optional[List[str]] = None					,							snake_case      :  str = "test"					,							**snake_case      :  Dict					,							):
								UpperCAmelCase_					:Optional[Any]      			=      gen_kwargs.copy()
								if gen_kwargs.get('''max_length'''     ) is None and gen_kwargs.get('''max_new_tokens'''     ) is None:
												UpperCAmelCase_					:Tuple      			=      self.args.generation_max_length
								UpperCAmelCase_					:Optional[Any]      			=      (
								    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''     ) is not None else self.args.generation_num_beams
								)
								UpperCAmelCase_					:Optional[Any]      			=      gen_kwargs

								return super().predict(snake_case					,							ignore_keys=snake_case					,							metric_key_prefix=snake_case     )





				def 				snake_case_ (							self      :  List[str]					,							snake_case      :  nn.Module					,							snake_case      :  Dict[str, Union[torch.Tensor, Any]]					,							snake_case      :  bool					,							snake_case      :  Optional[List[str]] = None					,							**snake_case      :  List[str]					,							):

								if not self.args.predict_with_generate or prediction_loss_only:
												return super().prediction_step(
												    snake_case					,							snake_case					,							prediction_loss_only=snake_case					,							ignore_keys=snake_case     )

								UpperCAmelCase_					:int      			=      '''labels''' in inputs
								UpperCAmelCase_					:Dict      			=      self._prepare_inputs(snake_case     )

								# XXX: adapt synced_gpus for fairscale as well
								# Priority (handled in generate):
								# gen_kwargs > model.generation_config > default GenerationConfig()

								if len(snake_case     ) == 0 and hasattr(self					,							'''_gen_kwargs'''     ):
												UpperCAmelCase_					:List[str]      			=      self._gen_kwargs.copy()

								if gen_kwargs.get('''max_length'''     ) is None and gen_kwargs.get('''max_new_tokens'''     ) is None:
												UpperCAmelCase_					:List[Any]      			=      self.model.config.max_length
								UpperCAmelCase_					:Optional[Any]      			=      (
								    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''     ) is not None else self.model.config.num_beams
								)
								UpperCAmelCase_					:Union[str, Any]      			=      True if is_deepspeed_zeroa_enabled() else False
								UpperCAmelCase_					:Any      			=      (
								    gen_kwargs['''synced_gpus'''] if gen_kwargs.get('''synced_gpus'''     ) is not None else default_synced_gpus
								)

								# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
								# (otherwise, it would continue generating from the padded `decoder_input_ids`)
								if (
								    "labels" in inputs
								    and "decoder_input_ids" in inputs
								    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
								):
												UpperCAmelCase_					:Union[str, Any]      			=      {k: v for k, v in inputs.items() if k != '''decoder_input_ids'''}
								UpperCAmelCase_					:str      			=      self.model.generate(**snake_case					,							**snake_case     )

								# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
								# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
								# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
								if self.model.generation_config._from_model_config:
												UpperCAmelCase_					:Dict      			=      False

								# Retrieves GenerationConfig from model.generation_config
								UpperCAmelCase_					:str      			=      self.model.generation_config
								# in case the batch is shorter than max length, the output should be padded
								if generated_tokens.shape[-1] < gen_config.max_length:
												UpperCAmelCase_					:Dict      			=      self._pad_tensors_to_max_len(snake_case					,							gen_config.max_length     )
								elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
												UpperCAmelCase_					:List[Any]      			=      self._pad_tensors_to_max_len(snake_case					,							gen_config.max_new_tokens + 1     )

								with torch.no_grad():
												if has_labels:
																with self.compute_loss_context_manager():
																				UpperCAmelCase_					:str      			=      model(**snake_case     )
																if self.label_smoother is not None:
																				UpperCAmelCase_					:str      			=      self.label_smoother(snake_case					,							inputs['''labels''']     ).mean().detach()
																else:
																				UpperCAmelCase_					:Dict      			=      (outputs['''loss'''] if isinstance(snake_case					,							snake_case     ) else outputs[0]).mean().detach()
												else:
																UpperCAmelCase_					:List[Any]      			=      None

								if self.args.prediction_loss_only:
												return loss, None, None

								if has_labels:
												UpperCAmelCase_					:int      			=      inputs['''labels''']
												if labels.shape[-1] < gen_config.max_length:
																UpperCAmelCase_					:Optional[Any]      			=      self._pad_tensors_to_max_len(snake_case					,							gen_config.max_length     )
												elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																UpperCAmelCase_					:Optional[Any]      			=      self._pad_tensors_to_max_len(snake_case					,							gen_config.max_new_tokens + 1     )
								else:
												UpperCAmelCase_					:Optional[int]      			=      None

								return loss, generated_tokens, labels





				def 				snake_case_ (							self      :  Dict					,							snake_case      :  Optional[Any]					,							snake_case      :  Union[str, Any]     ):
								if self.tokenizer is not None and hasattr(self.tokenizer					,							'''pad_token_id'''     ):
												# If PAD token is not defined at least EOS token has to be defined
												UpperCAmelCase_					:Dict      			=      (
												    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
												)
								else:
												if self.model.config.pad_token_id is not None:
																UpperCAmelCase_					:List[Any]      			=      self.model.config.pad_token_id
												else:
																raise ValueError('''Pad_token_id must be set in the configuration of the model, in order to pad tensors'''     )

								UpperCAmelCase_					:Optional[int]      			=      pad_token_id * torch.ones(
								    (tensor.shape[0], max_length)					,							dtype=tensor.dtype					,							device=tensor.device     )
								UpperCAmelCase_					:Union[str, Any]      			=      tensor
								return padded_tensor








"""simple docstring"""

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCamelCase								=						logging.get_logger(__name__)

__lowerCamelCase								=						{
    "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json",
}



class  _snake_case				(							A__       ):

				'''simple docstring'''




				UpperCamelCase__		     ="""mega"""





				def __init__(							self      :  Optional[int]					,							snake_case      :  Union[str, Any]=30_522					,							snake_case      :  Optional[Any]=128					,							snake_case      :  str=4					,							snake_case      :  Dict=256					,							snake_case      :  Optional[Any]=16					,							snake_case      :  Any=True					,							snake_case      :  str=64					,							snake_case      :  Optional[Any]=False					,							snake_case      :  Union[str, Any]=-1					,							snake_case      :  List[str]=None					,							snake_case      :  Any=True					,							snake_case      :  Optional[int]="scalenorm"					,							snake_case      :  Tuple=True					,							snake_case      :  str="silu"					,							snake_case      :  Dict="softmax"					,							snake_case      :  Optional[int]=0.1					,							snake_case      :  Tuple=0.1					,							snake_case      :  Dict=0.1					,							snake_case      :  List[str]=False					,							snake_case      :  Any=True					,							snake_case      :  List[Any]=256					,							snake_case      :  Optional[Any]=True					,							snake_case      :  Optional[int]=0.1					,							snake_case      :  str=2_048					,							snake_case      :  Union[str, Any]=False					,							snake_case      :  List[Any]=2					,							snake_case      :  str=0.02					,							snake_case      :  str=0.2					,							snake_case      :  str=0.02					,							snake_case      :  Union[str, Any]=1.0					,							snake_case      :  Optional[Any]=1					,							snake_case      :  List[Any]=0					,							snake_case      :  List[Any]=2					,							snake_case      :  List[Any]="rotary"					,							snake_case      :  Tuple=None					,							snake_case      :  int=True					,							snake_case      :  Dict=True					,							**snake_case      :  List[str]					,							):
								super().__init__(pad_token_id=snake_case					,							bos_token_id=snake_case					,							eos_token_id=snake_case					,							**snake_case     )

								UpperCAmelCase_					:List[Any]      			=      vocab_size
								UpperCAmelCase_					:Union[str, Any]      			=      hidden_size
								UpperCAmelCase_					:str      			=      num_hidden_layers
								UpperCAmelCase_					:List[Any]      			=      activation
								UpperCAmelCase_					:int      			=      attention_activation
								UpperCAmelCase_					:Tuple      			=      intermediate_size
								UpperCAmelCase_					:Optional[Any]      			=      ema_projection_size
								UpperCAmelCase_					:Optional[int]      			=      bidirectional
								UpperCAmelCase_					:Optional[int]      			=      shared_representation_size
								UpperCAmelCase_					:Optional[Any]      			=      use_chunking
								UpperCAmelCase_					:List[str]      			=      chunk_size
								UpperCAmelCase_					:Optional[Any]      			=      truncation
								UpperCAmelCase_					:List[str]      			=      normalize_before_mega
								UpperCAmelCase_					:List[Any]      			=      normalization_type
								UpperCAmelCase_					:Tuple      			=      norm_affine
								UpperCAmelCase_					:Union[str, Any]      			=      dropout_prob
								UpperCAmelCase_					:Tuple      			=      hidden_dropout_prob
								UpperCAmelCase_					:List[Any]      			=      attention_probs_dropout_prob
								UpperCAmelCase_					:str      			=      use_feature_dropout
								UpperCAmelCase_					:List[Any]      			=      use_normalized_ffn
								UpperCAmelCase_					:Any      			=      nffn_hidden_size
								UpperCAmelCase_					:List[Any]      			=      normalize_before_ffn
								UpperCAmelCase_					:Optional[int]      			=      nffn_activation_dropout_prob
								UpperCAmelCase_					:str      			=      max_positions
								UpperCAmelCase_					:int      			=      add_token_type_embeddings
								UpperCAmelCase_					:List[Any]      			=      type_vocab_size
								UpperCAmelCase_					:List[str]      			=      initializer_range
								UpperCAmelCase_					:int      			=      ema_delta_alpha_range
								UpperCAmelCase_					:Optional[int]      			=      ema_beta_range
								UpperCAmelCase_					:int      			=      ema_gamma_omega_range
								UpperCAmelCase_					:int      			=      relative_positional_bias
								UpperCAmelCase_					:List[Any]      			=      use_cache
								UpperCAmelCase_					:Optional[Any]      			=      classifier_dropout
								UpperCAmelCase_					:Optional[Any]      			=      add_lm_hidden_dense_layer
								UpperCAmelCase_					:str      			=      1  # not used but required by Hugging Face



class  _snake_case				(							A__       ):

				'''simple docstring'''




				@property
				def 				snake_case_ (							self      :  int     ):
								if self.task == "multiple-choice":
												UpperCAmelCase_					:List[str]      			=      {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
								else:
												UpperCAmelCase_					:Tuple      			=      {0: '''batch''', 1: '''sequence'''}
								return OrderedDict(
								    [
								        ('''input_ids''', dynamic_axis),
								        ('''attention_mask''', dynamic_axis),
								    ]     )


