
import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


A  :			Union[str, Any]     				=				logging.getLogger(__name__)







class     __A(				a    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case	,  _snake_case=None	,  _snake_case=None       )					->   str:


								'''simple docstring'''




								if current_layer == 0:
												__a    							=					self.embedding_hidden_mapping_in(_snake_case       )
								else:
												__a    							=					hidden_states[0]

								__a    							=					int(self.config.num_hidden_layers / self.config.num_hidden_groups       )

								# Index of the hidden group
								__a    							=					int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)       )

								__a    							=					self.albert_layer_groups[group_idx](
								    _snake_case	,  _snake_case	,  head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]	,  )
								__a    							=					layer_group_output[0]

								return (hidden_states,)







@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''						,     a						,     )
class     __A(				a    ):




				def __init__(  self	,  _snake_case       )					->   str:


								'''simple docstring'''




								super().__init__(_snake_case       )

								__a    							=					AlbertTransformerWithPabee(_snake_case       )

								self.init_weights()
								__a    							=					0
								__a    							=					0
								__a    							=					0

								__a    							=					0




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Union[str, Any]:


								'''simple docstring'''




								__a    							=					threshold




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   List[Any]:


								'''simple docstring'''




								__a    							=					patience




				def        SCREAMING_SNAKE_CASE_	(  self       )					->   List[Any]:


								'''simple docstring'''




								__a    							=					0
								__a    							=					0




				def        SCREAMING_SNAKE_CASE_	(  self       )					->   List[str]:


								'''simple docstring'''




								__a    							=					self.inference_layers_num / self.inference_instances_num
								__a    							=					(
								    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
								    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
								)
								print(_snake_case       )




				@add_start_docstrings_to_model_forward(_snake_case       )
				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=False	,  )					->   Any:


								'''simple docstring'''




								if input_ids is not None and inputs_embeds is not None:
												raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''       )
								elif input_ids is not None:
												__a    							=					input_ids.size()
								elif inputs_embeds is not None:
												__a    							=					inputs_embeds.size()[:-1]
								else:
												raise ValueError('''You have to specify either input_ids or inputs_embeds'''       )

								__a    							=					input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
												__a    							=					torch.ones(_snake_case	,  device=_snake_case       )
								if token_type_ids is None:
												__a    							=					torch.zeros(_snake_case	,  dtype=torch.long	,  device=_snake_case       )

								__a    							=					attention_mask.unsqueeze(1       ).unsqueeze(2       )
								__a    							=					extended_attention_mask.to(dtype=self.dtype       )  # fp16 compatibility
								__a    							=					(1.0 - extended_attention_mask) * -10_000.0
								__a    							=					self.get_head_mask(_snake_case	,  self.config.num_hidden_layers       )

								__a    							=					self.embeddings(
								    _snake_case	,  position_ids=_snake_case	,  token_type_ids=_snake_case	,  inputs_embeds=_snake_case       )
								__a    							=					embedding_output

								if self.training:
												__a    							=					[]
												for i in range(self.config.num_hidden_layers       ):
																__a    							=					self.encoder.adaptive_forward(
																    _snake_case	,  current_layer=_snake_case	,  attention_mask=_snake_case	,  head_mask=_snake_case	,  )

																__a    							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]       )       )
																__a    							=					output_layers[i](output_dropout(_snake_case       )       )
																res.append(_snake_case       )
								elif self.patience == 0:  # Use all layers for inference
												__a    							=					self.encoder(_snake_case	,  _snake_case	,  head_mask=_snake_case       )
												__a    							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]       )       )
												__a    							=					[output_layers[self.config.num_hidden_layers - 1](_snake_case       )]
								else:
												__a    							=					0
												__a    							=					None
												__a    							=					0
												for i in range(self.config.num_hidden_layers       ):
																calculated_layer_num += 1
																__a    							=					self.encoder.adaptive_forward(
																    _snake_case	,  current_layer=_snake_case	,  attention_mask=_snake_case	,  head_mask=_snake_case	,  )

																__a    							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]       )       )
																__a    							=					output_layers[i](_snake_case       )
																if regression:
																				__a    							=					logits.detach()
																				if patient_result is not None:
																								__a    							=					patient_result.detach()
																				if (patient_result is not None) and torch.abs(patient_result - labels       ) < self.regression_threshold:
																								patient_counter += 1
																				else:
																								__a    							=					0
																else:
																				__a    							=					logits.detach().argmax(dim=1       )
																				if patient_result is not None:
																								__a    							=					patient_result.detach().argmax(dim=1       )
																				if (patient_result is not None) and torch.all(labels.eq(_snake_case       )       ):
																								patient_counter += 1
																				else:
																								__a    							=					0

																__a    							=					logits
																if patient_counter == self.patience:
																				break
												__a    							=					[patient_result]
												self.inference_layers_num += calculated_layer_num
												self.inference_instances_num += 1

								return res







@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''						,     a						,     )
class     __A(				a    ):




				def __init__(  self	,  _snake_case       )					->   Any:


								'''simple docstring'''




								super().__init__(_snake_case       )
								__a    							=					config.num_labels

								__a    							=					AlbertModelWithPabee(_snake_case       )
								__a    							=					nn.Dropout(config.classifier_dropout_prob       )
								__a    							=					nn.ModuleList(
								    [nn.Linear(config.hidden_size	,  self.config.num_labels       ) for _ in range(config.num_hidden_layers       )]       )

								self.init_weights()




				@add_start_docstrings_to_model_forward(_snake_case       )
				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  _snake_case=None	,  )					->   Tuple:


								'''simple docstring'''




								__a    							=					self.albert(
								    input_ids=_snake_case	,  attention_mask=_snake_case	,  token_type_ids=_snake_case	,  position_ids=_snake_case	,  head_mask=_snake_case	,  inputs_embeds=_snake_case	,  output_dropout=self.dropout	,  output_layers=self.classifiers	,  regression=self.num_labels == 1	,  )

								__a    							=					(logits[-1],)

								if labels is not None:
												__a    							=					None
												__a    							=					0
												for ix, logits_item in enumerate(_snake_case       ):
																if self.num_labels == 1:
																				#  We are doing regression
																				__a    							=					MSELoss()
																				__a    							=					loss_fct(logits_item.view(-1       )	,  labels.view(-1       )       )
																else:
																				__a    							=					CrossEntropyLoss()
																				__a    							=					loss_fct(logits_item.view(-1	,  self.num_labels       )	,  labels.view(-1       )       )
																if total_loss is None:
																				__a    							=					loss
																else:
																				total_loss += loss * (ix + 1)
																total_weights += ix + 1
												__a    							=					(total_loss / total_weights,) + outputs

								return outputs

def 					__lowerCAmelCase    (    a__ = 99_8001						) ->						int:
				__a    							=					0
				for i in range(999		,					99		,					-1						):  # 3 digit numbers range from 999 down to 100
								for j in range(999		,					99		,					-1						):
												__a    							=					str(i * j						)
												if product_string == product_string[::-1] and i * j < n:
																__a    							=					max(a__		,					i * j						)
				return answer


if __name__ == "__main__":
					print(F"{solution() = }")