





"""simple docstring"""
import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
   import transformer_engine.pytorch as te




def 			__snake_case						(      __A ,__A=True ,__A=True ,__A=True    )		->				Union[str, Any]:
   if not is_fpa_available():
      raise ImportError("""Using `convert_model` requires transformer_engine to be installed."""    )
   for name, module in model.named_children():
      if isinstance(__A ,nn.Linear    ) and to_transformer_engine and _convert_linear:
         # Return early if the linear layer weights are not multiples of 16
         if any(p % 16 != 0 for p in module.weight.shape    ):
            return
         lowercase    :			Optional[int]  =       module.bias is not None
         lowercase    :			Union[str, Any]  =       te.Linear(
             module.in_features ,module.out_features ,bias=__A ,params_dtype=module.weight.dtype    )
         lowercase    :			Union[str, Any]  =       module.weight.data.clone()
         if has_bias:
            lowercase    :			Dict  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,nn.LayerNorm    ) and to_transformer_engine and _convert_ln:
         lowercase    :			List[str]  =       te.LayerNorm(module.normalized_shape[0] ,eps=module.eps ,params_dtype=module.weight.dtype    )
         lowercase    :			Optional[Any]  =       module.weight.data.clone()
         lowercase    :			Any  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,te.Linear    ) and not to_transformer_engine and _convert_linear:
         lowercase    :			Any  =       module.bias is not None
         lowercase    :			List[Any]  =       nn.Linear(
             module.in_features ,module.out_features ,bias=__A ,params_dtype=module.weight.dtype    )
         lowercase    :			Union[str, Any]  =       module.weight.data.clone()
         if has_bias:
            lowercase    :			Optional[Any]  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,te.LayerNorm    ) and not to_transformer_engine and _convert_ln:
         lowercase    :			Any  =       nn.LayerNorm(module.normalized_shape[0] ,eps=module.eps ,params_dtype=module.weight.dtype    )
         lowercase    :			str  =       module.weight.data.clone()
         lowercase    :			List[str]  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      else:
         convert_model(
             __A ,to_transformer_engine=__A ,_convert_linear=_convert_linear ,_convert_ln=_convert_ln ,)







def 			__snake_case						(      __A    )		->				Union[str, Any]:
   if not is_fpa_available():
      raise ImportError("""Using `has_transformer_engine_layers` requires transformer_engine to be installed."""    )
   for m in model.modules():
      if isinstance(__A ,(te.LayerNorm, te.Linear)    ):
         return True
   return False









"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase:	int							      =logging.get_logger(__name__)

lowerCAmelCase:	int							      ={
    "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/config.json",
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}
class lowerCamelCase__  (    __UpperCamelCase				):
   __UpperCAmelCase        =					"""qdqbert"""



   def __init__(       self      ,   snake_case=3_0_5_2_2      ,   snake_case=7_6_8      ,   snake_case=1_2      ,   snake_case=1_2      ,   snake_case=3_0_7_2      ,   snake_case="gelu"      ,   snake_case=0.1      ,   snake_case=0.1      ,   snake_case=5_1_2      ,   snake_case=2      ,   snake_case=0.02      ,   snake_case=1E-12      ,   snake_case=True      ,   snake_case=1      ,   snake_case=0      ,   snake_case=2      ,   **snake_case      ,   ) ->    List[str]:







      """simple docstring"""
      super().__init__(pad_token_id=snake_case      ,   bos_token_id=snake_case      ,   eos_token_id=snake_case      ,   **snake_case	)

      lowercase    :			Optional[Any]  =       vocab_size
      lowercase    :			int  =       max_position_embeddings
      lowercase    :			Dict  =       hidden_size
      lowercase    :			Union[str, Any]  =       num_hidden_layers
      lowercase    :			str  =       num_attention_heads
      lowercase    :			List[str]  =       intermediate_size
      lowercase    :			List[str]  =       hidden_act
      lowercase    :			List[str]  =       hidden_dropout_prob
      lowercase    :			Optional[Any]  =       attention_probs_dropout_prob
      lowercase    :			Optional[Any]  =       initializer_range
      lowercase    :			str  =       type_vocab_size
      lowercase    :			Optional[int]  =       layer_norm_eps
      lowercase    :			Optional[int]  =       use_cache


