


from ...configuration_utils import PretrainedConfig
from ...utils import logging


A :	Optional[Any]												=     logging.get_logger(__name__)

A :	Union[str, Any]												=     {
    'facebook/opt-125m': 'https://huggingface.co/facebook/opt-125m/blob/main/config.json',
    'facebook/opt-350m': 'https://huggingface.co/facebook/opt-350m/blob/main/config.json',
    'facebook/opt-1.3b': 'https://huggingface.co/facebook/opt-1.3b/blob/main/config.json',
    'facebook/opt-2.7b': 'https://huggingface.co/facebook/opt-2.7b/blob/main/config.json',
    'facebook/opt-6.7b': 'https://huggingface.co/facebook/opt-6.7b/blob/main/config.json',
    'facebook/opt-13b': 'https://huggingface.co/facebook/opt-13b/blob/main/config.json',
}



class 							A					( UpperCAmelCase__							):
					'''simple docstring'''






					A__    =				'''opt'''
					A__    =				['''past_key_values''']

					def __init__(self					:		Any   ,      _UpperCAmelCase					:		int=5_0272   ,      _UpperCAmelCase					:		str=768   ,      _UpperCAmelCase					:		Dict=12   ,      _UpperCAmelCase					:		str=3072   ,      _UpperCAmelCase					:		List[str]=2048   ,      _UpperCAmelCase					:		Tuple=True   ,      _UpperCAmelCase					:		str=False   ,      _UpperCAmelCase					:		str=None   ,      _UpperCAmelCase					:		Union[str, Any]=0.1   ,      _UpperCAmelCase					:		int=0.0   ,      _UpperCAmelCase					:		Tuple=12   ,      _UpperCAmelCase					:		Dict="relu"   ,      _UpperCAmelCase					:		Optional[int]=0.0   ,      _UpperCAmelCase					:		Any=0.02   ,      _UpperCAmelCase					:		Dict=True   ,      _UpperCAmelCase					:		List[Any]=1   ,      _UpperCAmelCase					:		List[str]=2   ,      _UpperCAmelCase					:		List[Any]=2   ,      _UpperCAmelCase					:		Any=True   ,      _UpperCAmelCase					:		int=True   ,      **_UpperCAmelCase					:		Optional[Any]   ,      )		->     Dict:




							"""simple docstring"""


							super().__init__(
							    pad_token_id=_UpperCAmelCase   ,      bos_token_id=_UpperCAmelCase   ,      eos_token_id=_UpperCAmelCase   ,      **_UpperCAmelCase   ,      )
							lowercase__      =	vocab_size
							lowercase__      =	max_position_embeddings
							lowercase__      =	num_attention_heads
							lowercase__      =	word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
							lowercase__      =	ffn_dim
							lowercase__      =	hidden_size
							lowercase__      =	num_hidden_layers
							lowercase__      =	dropout
							lowercase__      =	attention_dropout
							lowercase__      =	activation_function
							lowercase__      =	init_std
							lowercase__      =	layerdrop
							lowercase__      =	use_cache
							lowercase__      =	do_layer_norm_before
							# We keep these variables at `True` for backward compatibility.
							lowercase__      =	enable_bias
							lowercase__      =	layer_norm_elementwise_affine

							# Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
							# with checkpoints that have been fine-tuned before transformers v4.20.1
							# see https://github.com/facebookresearch/metaseq/pull/164
							lowercase__      =	_remove_final_layer_norm





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


A :	int												=     {
    'configuration_rwkv': ['RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RwkvConfig', 'RwkvOnnxConfig'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A :	List[Any]												=     [
	    'RWKV_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'RwkvForCausalLM',
	    'RwkvModel',
	    'RwkvPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_rwkv import RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP, RwkvConfig, RwkvOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_rwkv import (
		    RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
		    RwkvForCausalLM,
		    RwkvModel,
		    RwkvPreTrainedModel,
		)
else:
	import sys

	A :	Optional[int]												=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

