

'''simple docstring'''



import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
  import transformer_engine.pytorch as te







def      a__ (						a__   ,					a__=True   ,					a__=True   ,					a__=True		):




     """simple docstring"""


     if not is_fpa_available():
          raise ImportError("""Using `convert_model` requires transformer_engine to be installed."""		)
     for name, module in model.named_children():
          if isinstance(a__   ,					nn.Linear		) and to_transformer_engine and _convert_linear:
               # Return early if the linear layer weights are not multiples of 16
               if any(p % 16 != 0 for p in module.weight.shape		):
                    return
               __SCREAMING_SNAKE_CASE						     =   module.bias is not None
               __SCREAMING_SNAKE_CASE						     =   te.Linear(
                   module.in_features   ,					module.out_features   ,					bias=a__   ,					params_dtype=module.weight.dtype		)
               __SCREAMING_SNAKE_CASE						     =   module.weight.data.clone()
               if has_bias:
                    __SCREAMING_SNAKE_CASE						     =   module.bias.data.clone()

               setattr(a__   ,					a__   ,					a__		)
          elif isinstance(a__   ,					nn.LayerNorm		) and to_transformer_engine and _convert_ln:
               __SCREAMING_SNAKE_CASE						     =   te.LayerNorm(module.normalized_shape[0]   ,					eps=module.eps   ,					params_dtype=module.weight.dtype		)
               __SCREAMING_SNAKE_CASE						     =   module.weight.data.clone()
               __SCREAMING_SNAKE_CASE						     =   module.bias.data.clone()

               setattr(a__   ,					a__   ,					a__		)
          elif isinstance(a__   ,					te.Linear		) and not to_transformer_engine and _convert_linear:
               __SCREAMING_SNAKE_CASE						     =   module.bias is not None
               __SCREAMING_SNAKE_CASE						     =   nn.Linear(
                   module.in_features   ,					module.out_features   ,					bias=a__   ,					params_dtype=module.weight.dtype		)
               __SCREAMING_SNAKE_CASE						     =   module.weight.data.clone()
               if has_bias:
                    __SCREAMING_SNAKE_CASE						     =   module.bias.data.clone()

               setattr(a__   ,					a__   ,					a__		)
          elif isinstance(a__   ,					te.LayerNorm		) and not to_transformer_engine and _convert_ln:
               __SCREAMING_SNAKE_CASE						     =   nn.LayerNorm(module.normalized_shape[0]   ,					eps=module.eps   ,					params_dtype=module.weight.dtype		)
               __SCREAMING_SNAKE_CASE						     =   module.weight.data.clone()
               __SCREAMING_SNAKE_CASE						     =   module.bias.data.clone()

               setattr(a__   ,					a__   ,					a__		)
          else:
               convert_model(
                   a__   ,					to_transformer_engine=a__   ,					_convert_linear=_convert_linear   ,					_convert_ln=_convert_ln   ,					)







def      a__ (						a__		):




     """simple docstring"""


     if not is_fpa_available():
          raise ImportError("""Using `has_transformer_engine_layers` requires transformer_engine to be installed."""		)
     for m in model.modules():
          if isinstance(a__   ,					(te.LayerNorm, te.Linear)		):
               return True
     return False









'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase				: Union[str, Any]         =  logging.get_logger(__name__)

UpperCAmelCase				: Optional[int]         =  {
    'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/config.json',
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}







class 		lowerCAmelCase__     (   a      ):



 """simple docstring"""



 lowerCAmelCase__						      =   "qdqbert"
 def __init__(	self      : str      ,							__SCREAMING_SNAKE_CASE      : Optional[int]=30_522      ,							__SCREAMING_SNAKE_CASE      : str=768      ,							__SCREAMING_SNAKE_CASE      : Union[str, Any]=12      ,							__SCREAMING_SNAKE_CASE      : Any=12      ,							__SCREAMING_SNAKE_CASE      : Any=3_072      ,							__SCREAMING_SNAKE_CASE      : List[str]="gelu"      ,							__SCREAMING_SNAKE_CASE      : Any=0.1      ,							__SCREAMING_SNAKE_CASE      : Any=0.1      ,							__SCREAMING_SNAKE_CASE      : Optional[int]=512      ,							__SCREAMING_SNAKE_CASE      : List[str]=2      ,							__SCREAMING_SNAKE_CASE      : Dict=0.02      ,							__SCREAMING_SNAKE_CASE      : List[str]=1E-12      ,							__SCREAMING_SNAKE_CASE      : str=True      ,							__SCREAMING_SNAKE_CASE      : Optional[int]=1      ,							__SCREAMING_SNAKE_CASE      : Optional[int]=0      ,							__SCREAMING_SNAKE_CASE      : Dict=2      ,							**__SCREAMING_SNAKE_CASE      : Union[str, Any]      ,							)						->				str:


      """simple docstring"""




      super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE      ,							bos_token_id=__SCREAMING_SNAKE_CASE      ,							eos_token_id=__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE    )

      __SCREAMING_SNAKE_CASE						     =   vocab_size
      __SCREAMING_SNAKE_CASE						     =   max_position_embeddings
      __SCREAMING_SNAKE_CASE						     =   hidden_size
      __SCREAMING_SNAKE_CASE						     =   num_hidden_layers
      __SCREAMING_SNAKE_CASE						     =   num_attention_heads
      __SCREAMING_SNAKE_CASE						     =   intermediate_size
      __SCREAMING_SNAKE_CASE						     =   hidden_act
      __SCREAMING_SNAKE_CASE						     =   hidden_dropout_prob
      __SCREAMING_SNAKE_CASE						     =   attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE						     =   initializer_range
      __SCREAMING_SNAKE_CASE						     =   type_vocab_size
      __SCREAMING_SNAKE_CASE						     =   layer_norm_eps
      __SCREAMING_SNAKE_CASE						     =   use_cache






