


import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


lowerCAmelCase__						:List[str]         =							logging.getLogger(__name__)

class 	__a   (       lowercase__    ):
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None		)				-> List[Any]:




									"""simple docstring"""





									if current_layer == 0:
																_UpperCAmelCase					=    self.embedding_hidden_mapping_in(_A		)
									else:
																_UpperCAmelCase					=    hidden_states[0]

									_UpperCAmelCase					=    int(self.config.num_hidden_layers / self.config.num_hidden_groups		)

									# Index of the hidden group
									_UpperCAmelCase					=    int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)		)

									_UpperCAmelCase					=    self.albert_layer_groups[group_idx](
									    _A    ,					_A    ,					head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]    ,					)
									_UpperCAmelCase					=    layer_group_output[0]

									return (hidden_states,)



@add_start_docstrings(
    'The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'					,   lowercase__					,   )
class 	__a   (       lowercase__    ):
		def __init__(    self    ,					_SCREAMING_SNAKE_CASE		)				-> List[str]:




									"""simple docstring"""





									super().__init__(_A		)

									_UpperCAmelCase					=    AlbertTransformerWithPabee(_A		)

									self.init_weights()
									_UpperCAmelCase					=    0
									_UpperCAmelCase					=    0
									_UpperCAmelCase					=    0

									_UpperCAmelCase					=    0
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE		)				-> Dict:




									"""simple docstring"""





									_UpperCAmelCase					=    threshold
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE		)				-> List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=    patience
		def 			UpperCAmelCase__       (    self		)				-> Dict:




									"""simple docstring"""





									_UpperCAmelCase					=    0
									_UpperCAmelCase					=    0
		def 			UpperCAmelCase__       (    self		)				-> Dict:




									"""simple docstring"""





									_UpperCAmelCase					=    self.inference_layers_num / self.inference_instances_num
									_UpperCAmelCase					=    (
									    f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
									    f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
									)
									print(_A		)






		@add_start_docstrings_to_model_forward(_A		)
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=False    ,					)				-> List[Any]:




									"""simple docstring"""





									if input_ids is not None and inputs_embeds is not None:
																raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'		)
									elif input_ids is not None:
																_UpperCAmelCase					=    input_ids.size()
									elif inputs_embeds is not None:
																_UpperCAmelCase					=    inputs_embeds.size()[:-1]
									else:
																raise ValueError('You have to specify either input_ids or inputs_embeds'		)

									_UpperCAmelCase					=    input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
																_UpperCAmelCase					=    torch.ones(_A    ,					device=_A		)
									if token_type_ids is None:
																_UpperCAmelCase					=    torch.zeros(_A    ,					dtype=torch.long    ,					device=_A		)

									_UpperCAmelCase					=    attention_mask.unsqueeze(1		).unsqueeze(2		)
									_UpperCAmelCase					=    extended_attention_mask.to(dtype=self.dtype		)  # fp16 compatibility
									_UpperCAmelCase					=    (1.0 - extended_attention_mask) * -10000.0
									_UpperCAmelCase					=    self.get_head_mask(_A    ,					self.config.num_hidden_layers		)

									_UpperCAmelCase					=    self.embeddings(
									    _A    ,					position_ids=_A    ,					token_type_ids=_A    ,					inputs_embeds=_A		)
									_UpperCAmelCase					=    embedding_output

									if self.training:
																_UpperCAmelCase					=    []
																for i in range(self.config.num_hidden_layers		):
																							_UpperCAmelCase					=    self.encoder.adaptive_forward(
																							    _A    ,					current_layer=_A    ,					attention_mask=_A    ,					head_mask=_A    ,					)

																							_UpperCAmelCase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
																							_UpperCAmelCase					=    output_layers[i](output_dropout(_A		)		)
																							res.append(_A		)
									elif self.patience == 0:  # Use all layers for inference
																_UpperCAmelCase					=    self.encoder(_A    ,					_A    ,					head_mask=_A		)
																_UpperCAmelCase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
																_UpperCAmelCase					=    [output_layers[self.config.num_hidden_layers - 1](_A		)]
									else:
																_UpperCAmelCase					=    0
																_UpperCAmelCase					=    None
																_UpperCAmelCase					=    0
																for i in range(self.config.num_hidden_layers		):
																							calculated_layer_num += 1
																							_UpperCAmelCase					=    self.encoder.adaptive_forward(
																							    _A    ,					current_layer=_A    ,					attention_mask=_A    ,					head_mask=_A    ,					)

																							_UpperCAmelCase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
																							_UpperCAmelCase					=    output_layers[i](_A		)
																							if regression:
																														_UpperCAmelCase					=    logits.detach()
																														if patient_result is not None:
																																					_UpperCAmelCase					=    patient_result.detach()
																														if (patient_result is not None) and torch.abs(patient_result - labels		) < self.regression_threshold:
																																					patient_counter += 1
																														else:
																																					_UpperCAmelCase					=    0
																							else:
																														_UpperCAmelCase					=    logits.detach().argmax(dim=1		)
																														if patient_result is not None:
																																					_UpperCAmelCase					=    patient_result.detach().argmax(dim=1		)
																														if (patient_result is not None) and torch.all(labels.eq(_A		)		):
																																					patient_counter += 1
																														else:
																																					_UpperCAmelCase					=    0

																							_UpperCAmelCase					=    logits
																							if patient_counter == self.patience:
																														break
																_UpperCAmelCase					=    [patient_result]
																self.inference_layers_num += calculated_layer_num
																self.inference_instances_num += 1

									return res



@add_start_docstrings(
    'Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '					,   lowercase__					,   )
class 	__a   (       lowercase__    ):
		def __init__(    self    ,					_SCREAMING_SNAKE_CASE		)				-> List[Any]:




									"""simple docstring"""





									super().__init__(_A		)
									_UpperCAmelCase					=    config.num_labels

									_UpperCAmelCase					=    AlbertModelWithPabee(_A		)
									_UpperCAmelCase					=    nn.Dropout(config.classifier_dropout_prob		)
									_UpperCAmelCase					=    nn.ModuleList(
									    [nn.Linear(config.hidden_size    ,					self.config.num_labels		) for _ in range(config.num_hidden_layers		)]		)

									self.init_weights()






		@add_start_docstrings_to_model_forward(_A		)
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					)				-> Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=    self.albert(
									    input_ids=_A    ,					attention_mask=_A    ,					token_type_ids=_A    ,					position_ids=_A    ,					head_mask=_A    ,					inputs_embeds=_A    ,					output_dropout=self.dropout    ,					output_layers=self.classifiers    ,					regression=self.num_labels == 1    ,					)

									_UpperCAmelCase					=    (logits[-1],)

									if labels is not None:
																_UpperCAmelCase					=    None
																_UpperCAmelCase					=    0
																for ix, logits_item in enumerate(_A		):
																							if self.num_labels == 1:
																														#  We are doing regression
																														_UpperCAmelCase					=    MSELoss()
																														_UpperCAmelCase					=    loss_fct(logits_item.view(-1		)    ,					labels.view(-1		)		)
																							else:
																														_UpperCAmelCase					=    CrossEntropyLoss()
																														_UpperCAmelCase					=    loss_fct(logits_item.view(-1    ,					self.num_labels		)    ,					labels.view(-1		)		)
																							if total_loss is None:
																														_UpperCAmelCase					=    loss
																							else:
																														total_loss += loss * (ix + 1)
																							total_weights += ix + 1
																_UpperCAmelCase					=    (total_loss / total_weights,) + outputs

									return outputs




import warnings

from .generation import GenerationMixin

class 	__a   (       UpperCAmelCase    ):
		# warning at import time
		warnings.warn(
		    'Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will '
		    'be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead.'					,   UpperCAmelCase					,   )
