





'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


A__       :     List[Any]          ={'''configuration_hubert''': ['''HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''HubertConfig''']}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__       :     Union[str, Any]          =[
          '''HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''HubertForCTC''',
          '''HubertForSequenceClassification''',
          '''HubertModel''',
          '''HubertPreTrainedModel''',
      ]


try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__       :     str          =[
          '''TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''TFHubertForCTC''',
          '''TFHubertModel''',
          '''TFHubertPreTrainedModel''',
      ]

if TYPE_CHECKING:
      from .configuration_hubert import HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, HubertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_hubert import (
                HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                HubertForCTC,
                HubertForSequenceClassification,
                HubertModel,
                HubertPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_hubert import (
                TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFHubertForCTC,
                TFHubertModel,
                TFHubertPreTrainedModel,
            )


else:
      import sys

      A__       :     Optional[Any]          =_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)













'''simple docstring'''






import math

import torch
from torch import nn
from torch.nn import init

from .binarizer import MagnitudeBinarizer, ThresholdBinarizer, TopKBinarizer




class 				UpperCAmelCase  (    nn.Linear     ):




      def __init__(    self    :				str    ,    __snake_case    :				int    ,    __snake_case    :				int    ,    __snake_case    :				bool = True    ,    __snake_case    :				str = "constant"    ,    __snake_case    :				float = 0.0    ,    __snake_case    :				str = "topK"    ,    )       ->   int:
            super(__snake_case    ,    self						).__init__(in_features=__snake_case    ,    out_features=__snake_case    ,    bias=__snake_case						)
            assert pruning_method in ["topK", "threshold", "sigmoied_threshold", "magnitude", "l0"]
            _lowerCAmelCase							  =							pruning_method

            if self.pruning_method in ["topK", "threshold", "sigmoied_threshold", "l0"]:
                  _lowerCAmelCase							  =							mask_scale
                  _lowerCAmelCase							  =							mask_init
                  _lowerCAmelCase							  =							nn.Parameter(torch.empty(self.weight.size()						)						)
                  self.init_mask()




      def        lowercase__					(    self    :				Any						)       ->   str:
            if self.mask_init == "constant":
                  init.constant_(self.mask_scores    ,    val=self.mask_scale						)
            elif self.mask_init == "uniform":
                  init.uniform_(self.mask_scores    ,    a=-self.mask_scale    ,    b=self.mask_scale						)
            elif self.mask_init == "kaiming":
                  init.kaiming_uniform_(self.mask_scores    ,    a=math.sqrt(5						)						)




      def        lowercase__					(    self    :				Dict    ,    __snake_case    :				torch.tensor    ,    __snake_case    :				float						)       ->   Union[str, Any]:
            # Get the mask
            if self.pruning_method == "topK":
                  _lowerCAmelCase							  =							TopKBinarizer.apply(self.mask_scores    ,    __snake_case						)
            elif self.pruning_method in ["threshold", "sigmoied_threshold"]:
                  _lowerCAmelCase							  =							"""sigmoied""" in self.pruning_method
                  _lowerCAmelCase							  =							ThresholdBinarizer.apply(self.mask_scores    ,    __snake_case    ,    __snake_case						)
            elif self.pruning_method == "magnitude":
                  _lowerCAmelCase							  =							MagnitudeBinarizer.apply(self.weight    ,    __snake_case						)
            elif self.pruning_method == "l0":
                  _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase							  =							-0.1, 1.1, 2 / 3
                  if self.training:
                        _lowerCAmelCase							  =							torch.zeros_like(self.mask_scores						).uniform_().clamp(0.00_01    ,    0.99_99						)
                        _lowerCAmelCase							  =							torch.sigmoid((u.log() - (1 - u).log() + self.mask_scores) / b						)
                  else:
                        _lowerCAmelCase							  =							torch.sigmoid(self.mask_scores						)
                  _lowerCAmelCase							  =							s * (r - l) + l
                  _lowerCAmelCase							  =							s_bar.clamp(min=0.0    ,    max=1.0						)
            # Mask weights with computed mask
            _lowerCAmelCase							  =							mask * self.weight
            # Compute output (linear layer) with masked weights
            return nn.functional.linear(__snake_case    ,    __snake_case    ,    self.bias						)






