


import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm


def     UpperCamelCase(						__UpperCamelCase							:	List[str]  ,__UpperCamelCase							:	Optional[int]				):

		if isinstance(__UpperCamelCase  ,(list, tuple)				):
				return honor_type(__UpperCamelCase  ,(move_to_device(__UpperCamelCase  ,__UpperCamelCase				) for t in state)				)
		elif isinstance(__UpperCamelCase  ,__UpperCamelCase				):
				return type(__UpperCamelCase				)({k: move_to_device(__UpperCamelCase  ,__UpperCamelCase				) for k, v in state.items()}				)
		elif isinstance(__UpperCamelCase  ,torch.Tensor				):
				return state.to(__UpperCamelCase				)
		return state





class        __snake_case		(  torch.optim.Optimizer		):




							def __init__( self  : List[str]       ,       A_  : str       ,       A_  : List[str]=True       ,       A_  : Union[str, Any]=None):
									lowerCAmelCase_    :  str             =				optimizer
									lowerCAmelCase_    :  Tuple             =				scaler
									lowerCAmelCase_    :  Any             =				AcceleratorState()
									lowerCAmelCase_    :  Optional[Any]             =				GradientState()
									lowerCAmelCase_    :  Any             =				device_placement
									lowerCAmelCase_    :  List[Any]             =				False
									lowerCAmelCase_    :  int             =				None

									# Handle device placement
									if device_placement:
											lowerCAmelCase_    :  Dict             =				self.optimizer.state_dict()
											if self.accelerator_state.distributed_type == DistributedType.TPU:
													xm.send_cpu_data_to_device(__A       ,       self.accelerator_state.device)
											else:
													lowerCAmelCase_    :  int             =				move_to_device(__A       ,       self.accelerator_state.device)
											self.optimizer.load_state_dict(__A)




							@property
							def      UpperCAmelCase__    ( self  : Any):
									return self.optimizer.state




							@state.setter
							def      UpperCAmelCase__    ( self  : Optional[Any]       ,       A_  : Union[str, Any]):
									lowerCAmelCase_    :  str             =				state




							@property
							def      UpperCAmelCase__    ( self  : Dict):
									return self.optimizer.param_groups




							@param_groups.setter
							def      UpperCAmelCase__    ( self  : Tuple       ,       A_  : Tuple):
									lowerCAmelCase_    :  Dict             =				param_groups




							@property
							def      UpperCAmelCase__    ( self  : Tuple):
									return self.optimizer.defaults




							@defaults.setter
							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : List[Any]):
									lowerCAmelCase_    :  str             =				defaults




							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : int):
									self.optimizer.add_param_group(__A)




							def      UpperCAmelCase__    ( self  : Any       ,       A_  : List[str]):







									if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
											xm.send_cpu_data_to_device(__A       ,       self.accelerator_state.device)
									self.optimizer.load_state_dict(__A)




							def      UpperCAmelCase__    ( self  : Tuple):
									return self.optimizer.state_dict()




							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : Tuple=None):







									if self.gradient_state.sync_gradients:
											lowerCAmelCase_    :  Optional[Any]             =				"set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters
											if accept_arg:
													if set_to_none is None:
															lowerCAmelCase_    :  Optional[int]             =				False
													self.optimizer.zero_grad(set_to_none=__A)
											else:
													if set_to_none is not None:
															raise ValueError('''`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.''')
													self.optimizer.zero_grad()




							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : Optional[Any]=None):







									if self.gradient_state.sync_gradients:
											if self.accelerator_state.distributed_type == DistributedType.TPU:
													lowerCAmelCase_    :  Optional[Any]             =				{"closure": closure} if closure is not None else {}
													xm.optimizer_step(self.optimizer       ,       optimizer_args=__A)
											elif self.scaler is not None:
													lowerCAmelCase_    :  Dict             =				False
													if self._last_scale is None:
															# `get_scale` is an async operation requiring full synchronization
															# on CPU and GPUs before finishing. As a result, we store away
															# the prior one to reduce the call overhead
															lowerCAmelCase_    :  Dict             =				self.scaler.get_scale()
															lowerCAmelCase_    :  List[Any]             =				True
													self.scaler.step(self.optimizer       ,       __A)
													self.scaler.update()
													lowerCAmelCase_    :  List[Any]             =				self.scaler.get_scale()
													if not new_scale:
															# If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
															lowerCAmelCase_    :  List[Any]             =				scale_after < self._last_scale
													lowerCAmelCase_    :  List[Any]             =				scale_after

											else:
													self.optimizer.step(__A)




							def      UpperCAmelCase__    ( self  : str       ,       A_  : Optional[Any]):







									for param_group in self.optimizer.param_groups:
											lowerCAmelCase_    :  str             =				[parameters_map.get(__A       ,       __A) for p in param_group["params"]]




							@property
							def      UpperCAmelCase__    ( self  : int):
									warnings.warn(
									    '''The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '''
									    '''`optimizer.step_was_skipped` instead.'''       ,       __A       ,       )
									return self._is_overflow




							@property
							def      UpperCAmelCase__    ( self  : Optional[int]):
									return self._is_overflow




							def __getstate__( self  : Union[str, Any]):
									return self.__dict__.copy()







							def __setstate__( self  : Dict       ,       A_  : int):
									self.__dict__.update(__A)










from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__						:						int          =      logging.get_logger(__name__)

A__						:						Dict          =      {
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}



class        __snake_case		(  UpperCamelCase_		):
							_a       =				'''mega'''




							def __init__( self  : Any       ,       A_  : Dict=3_0_5_2_2       ,       A_  : int=1_2_8       ,       A_  : str=4       ,       A_  : int=2_5_6       ,       A_  : int=1_6       ,       A_  : str=True       ,       A_  : List[str]=6_4       ,       A_  : Optional[Any]=False       ,       A_  : str=-1       ,       A_  : Optional[int]=None       ,       A_  : Tuple=True       ,       A_  : Tuple="scalenorm"       ,       A_  : Optional[Any]=True       ,       A_  : int="silu"       ,       A_  : Optional[int]="softmax"       ,       A_  : Optional[int]=0.1       ,       A_  : Union[str, Any]=0.1       ,       A_  : Any=0.1       ,       A_  : Dict=False       ,       A_  : int=True       ,       A_  : Tuple=2_5_6       ,       A_  : int=True       ,       A_  : List[Any]=0.1       ,       A_  : Optional[Any]=2_0_4_8       ,       A_  : Dict=False       ,       A_  : Dict=2       ,       A_  : Tuple=0.02       ,       A_  : List[Any]=0.2       ,       A_  : str=0.02       ,       A_  : List[str]=1.0       ,       A_  : Union[str, Any]=1       ,       A_  : List[Any]=0       ,       A_  : int=2       ,       A_  : Tuple="rotary"       ,       A_  : Any=None       ,       A_  : Tuple=True       ,       A_  : Any=True       ,       **A_  : Optional[int]       ,       ):
									super().__init__(pad_token_id=A_       ,       bos_token_id=A_       ,       eos_token_id=A_       ,       **A_)

									lowerCAmelCase_    :  Tuple             =				vocab_size
									lowerCAmelCase_    :  str             =				hidden_size
									lowerCAmelCase_    :  Any             =				num_hidden_layers
									lowerCAmelCase_    :  Optional[int]             =				activation
									lowerCAmelCase_    :  Any             =				attention_activation
									lowerCAmelCase_    :  List[Any]             =				intermediate_size
									lowerCAmelCase_    :  Any             =				ema_projection_size
									lowerCAmelCase_    :  Optional[int]             =				bidirectional
									lowerCAmelCase_    :  Tuple             =				shared_representation_size
									lowerCAmelCase_    :  Any             =				use_chunking
									lowerCAmelCase_    :  List[str]             =				chunk_size
									lowerCAmelCase_    :  int             =				truncation
									lowerCAmelCase_    :  Tuple             =				normalize_before_mega
									lowerCAmelCase_    :  Tuple             =				normalization_type
									lowerCAmelCase_    :  List[str]             =				norm_affine
									lowerCAmelCase_    :  List[str]             =				dropout_prob
									lowerCAmelCase_    :  List[str]             =				hidden_dropout_prob
									lowerCAmelCase_    :  Tuple             =				attention_probs_dropout_prob
									lowerCAmelCase_    :  str             =				use_feature_dropout
									lowerCAmelCase_    :  Union[str, Any]             =				use_normalized_ffn
									lowerCAmelCase_    :  Union[str, Any]             =				nffn_hidden_size
									lowerCAmelCase_    :  int             =				normalize_before_ffn
									lowerCAmelCase_    :  Tuple             =				nffn_activation_dropout_prob
									lowerCAmelCase_    :  List[str]             =				max_positions
									lowerCAmelCase_    :  Union[str, Any]             =				add_token_type_embeddings
									lowerCAmelCase_    :  str             =				type_vocab_size
									lowerCAmelCase_    :  int             =				initializer_range
									lowerCAmelCase_    :  str             =				ema_delta_alpha_range
									lowerCAmelCase_    :  Tuple             =				ema_beta_range
									lowerCAmelCase_    :  Any             =				ema_gamma_omega_range
									lowerCAmelCase_    :  Any             =				relative_positional_bias
									lowerCAmelCase_    :  Optional[int]             =				use_cache
									lowerCAmelCase_    :  int             =				classifier_dropout
									lowerCAmelCase_    :  List[Any]             =				add_lm_hidden_dense_layer
									lowerCAmelCase_    :  Optional[Any]             =				1  # not used but required by Hugging Face





class        __snake_case		(  UpperCamelCase_		):




							@property
							def      UpperCAmelCase__    ( self  : Any):
									if self.task == "multiple-choice":
											lowerCAmelCase_    :  Dict             =				{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
									else:
											lowerCAmelCase_    :  int             =				{0: '''batch''', 1: '''sequence'''}
									return OrderedDict(
									    [
									        ('''input_ids''', dynamic_axis),
									        ('''attention_mask''', dynamic_axis),
									    ])





