




'''simple docstring'''


import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


lowerCAmelCase_			:     Tuple							    =     logging.getLogger(__name__)







class     SCREAMING_SNAKE_CASE						(	snake_case_							):






      def        lowercase_	(   self      :     int	,	lowercase__      :     str	,	lowercase__      :     str	,	lowercase__      :     Any=None	,	lowercase__      :     str=None					):


           '''simple docstring'''

           if current_layer == 0:
                a_    :       List[Any]    	=     self.embedding_hidden_mapping_in(lowercase__					)
           else:
                a_    :       List[str]    	=     hidden_states[0]

           a_    :       Tuple    	=     int(self.config.num_hidden_layers / self.config.num_hidden_groups					)

           # Index of the hidden group
           a_    :       Optional[Any]    	=     int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)					)

           a_    :       int    	=     self.albert_layer_groups[group_idx](
               lowercase__	,	lowercase__	,	head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]	,	)
           a_    :       int    	=     layer_group_output[0]

           return (hidden_states,)







@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''       ,			snake_case_       ,			)
class     SCREAMING_SNAKE_CASE						(	snake_case_							):






      def __init__(   self      :     int	,	lowercase__      :     Optional[int]					):


           '''simple docstring'''

           super().__init__(lowercase__					)

           a_    :       str    	=     AlbertTransformerWithPabee(lowercase__					)

           self.init_weights()
           a_    :       Any    	=     0
           a_    :       Optional[Any]    	=     0
           a_    :       Optional[int]    	=     0

           a_    :       int    	=     0






      def        lowercase_	(   self      :     Union[str, Any]	,	lowercase__      :     List[str]					):


           '''simple docstring'''

           a_    :       List[Any]    	=     threshold






      def        lowercase_	(   self      :     str	,	lowercase__      :     Any					):


           '''simple docstring'''

           a_    :       str    	=     patience






      def        lowercase_	(   self      :     Optional[Any]					):


           '''simple docstring'''

           a_    :       int    	=     0
           a_    :       Tuple    	=     0






      def        lowercase_	(   self      :     Dict					):


           '''simple docstring'''

           a_    :       Optional[Any]    	=     self.inference_layers_num / self.inference_instances_num
           a_    :       int    	=     (
               F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
               F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
           )
           print(lowercase__					)






      @add_start_docstrings_to_model_forward(lowercase__					)
      def        lowercase_	(   self      :     List[str]	,	lowercase__      :     Dict=None	,	lowercase__      :     Optional[Any]=None	,	lowercase__      :     Any=None	,	lowercase__      :     int=None	,	lowercase__      :     Union[str, Any]=None	,	lowercase__      :     Tuple=None	,	lowercase__      :     int=None	,	lowercase__      :     Dict=None	,	lowercase__      :     Optional[Any]=False	,	):


           '''simple docstring'''

           if input_ids is not None and inputs_embeds is not None:
                raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""					)
           elif input_ids is not None:
                a_    :       Tuple    	=     input_ids.size()
           elif inputs_embeds is not None:
                a_    :       List[str]    	=     inputs_embeds.size()[:-1]
           else:
                raise ValueError("""You have to specify either input_ids or inputs_embeds"""					)

           a_    :       Dict    	=     input_ids.device if input_ids is not None else inputs_embeds.device

           if attention_mask is None:
                a_    :       List[Any]    	=     torch.ones(lowercase__	,	device=lowercase__					)
           if token_type_ids is None:
                a_    :       str    	=     torch.zeros(lowercase__	,	dtype=torch.long	,	device=lowercase__					)

           a_    :       int    	=     attention_mask.unsqueeze(1					).unsqueeze(2					)
           a_    :       List[str]    	=     extended_attention_mask.to(dtype=self.dtype					)  # fp16 compatibility
           a_    :       Dict    	=     (1.0 - extended_attention_mask) * -10000.0
           a_    :       List[str]    	=     self.get_head_mask(lowercase__	,	self.config.num_hidden_layers					)

           a_    :       int    	=     self.embeddings(
               lowercase__	,	position_ids=lowercase__	,	token_type_ids=lowercase__	,	inputs_embeds=lowercase__					)
           a_    :       int    	=     embedding_output

           if self.training:
                a_    :       List[Any]    	=     []
                for i in range(self.config.num_hidden_layers					):
                     a_    :       Any    	=     self.encoder.adaptive_forward(
                         lowercase__	,	current_layer=lowercase__	,	attention_mask=lowercase__	,	head_mask=lowercase__	,	)

                     a_    :       Dict    	=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
                     a_    :       Union[str, Any]    	=     output_layers[i](output_dropout(lowercase__					)					)
                     res.append(lowercase__					)
           elif self.patience == 0:  # Use all layers for inference
                a_    :       int    	=     self.encoder(lowercase__	,	lowercase__	,	head_mask=lowercase__					)
                a_    :       str    	=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
                a_    :       Optional[int]    	=     [output_layers[self.config.num_hidden_layers - 1](lowercase__					)]
           else:
                a_    :       Dict    	=     0
                a_    :       List[str]    	=     None
                a_    :       Dict    	=     0
                for i in range(self.config.num_hidden_layers					):
                     calculated_layer_num += 1
                     a_    :       str    	=     self.encoder.adaptive_forward(
                         lowercase__	,	current_layer=lowercase__	,	attention_mask=lowercase__	,	head_mask=lowercase__	,	)

                     a_    :       int    	=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
                     a_    :       Optional[int]    	=     output_layers[i](lowercase__					)
                     if regression:
                          a_    :       Optional[Any]    	=     logits.detach()
                          if patient_result is not None:
                               a_    :       Dict    	=     patient_result.detach()
                          if (patient_result is not None) and torch.abs(patient_result - labels					) < self.regression_threshold:
                               patient_counter += 1
                          else:
                               a_    :       List[Any]    	=     0
                     else:
                          a_    :       List[str]    	=     logits.detach().argmax(dim=1					)
                          if patient_result is not None:
                               a_    :       Optional[Any]    	=     patient_result.detach().argmax(dim=1					)
                          if (patient_result is not None) and torch.all(labels.eq(lowercase__					)					):
                               patient_counter += 1
                          else:
                               a_    :       Union[str, Any]    	=     0

                     a_    :       Union[str, Any]    	=     logits
                     if patient_counter == self.patience:
                          break
                a_    :       Optional[Any]    	=     [patient_result]
                self.inference_layers_num += calculated_layer_num
                self.inference_instances_num += 1

           return res







@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''       ,			snake_case_       ,			)
class     SCREAMING_SNAKE_CASE						(	snake_case_							):






      def __init__(   self      :     str	,	lowercase__      :     int					):


           '''simple docstring'''

           super().__init__(lowercase__					)
           a_    :       Any    	=     config.num_labels

           a_    :       Optional[int]    	=     AlbertModelWithPabee(lowercase__					)
           a_    :       Union[str, Any]    	=     nn.Dropout(config.classifier_dropout_prob					)
           a_    :       Optional[int]    	=     nn.ModuleList(
               [nn.Linear(config.hidden_size	,	self.config.num_labels					) for _ in range(config.num_hidden_layers					)]					)

           self.init_weights()






      @add_start_docstrings_to_model_forward(lowercase__					)
      def        lowercase_	(   self      :     Any	,	lowercase__      :     Dict=None	,	lowercase__      :     Any=None	,	lowercase__      :     Tuple=None	,	lowercase__      :     Union[str, Any]=None	,	lowercase__      :     Union[str, Any]=None	,	lowercase__      :     str=None	,	lowercase__      :     int=None	,	):


           '''simple docstring'''

           a_    :       Union[str, Any]    	=     self.albert(
               input_ids=lowercase__	,	attention_mask=lowercase__	,	token_type_ids=lowercase__	,	position_ids=lowercase__	,	head_mask=lowercase__	,	inputs_embeds=lowercase__	,	output_dropout=self.dropout	,	output_layers=self.classifiers	,	regression=self.num_labels == 1	,	)

           a_    :       Dict    	=     (logits[-1],)

           if labels is not None:
                a_    :       Tuple    	=     None
                a_    :       int    	=     0
                for ix, logits_item in enumerate(lowercase__					):
                     if self.num_labels == 1:
                          #  We are doing regression
                          a_    :       Dict    	=     MSELoss()
                          a_    :       Union[str, Any]    	=     loss_fct(logits_item.view(-1					)	,	labels.view(-1					)					)
                     else:
                          a_    :       str    	=     CrossEntropyLoss()
                          a_    :       Union[str, Any]    	=     loss_fct(logits_item.view(-1	,	self.num_labels					)	,	labels.view(-1					)					)
                     if total_loss is None:
                          a_    :       Any    	=     loss
                     else:
                          total_loss += loss * (ix + 1)
                     total_weights += ix + 1
                a_    :       List[Any]    	=     (total_loss / total_weights,) + outputs

           return outputs






'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import CLIPImageProcessor







class     SCREAMING_SNAKE_CASE						(	unittest.TestCase							):






      def __init__(   self      :     Tuple	,	lowercase__      :     Union[str, Any]	,	lowercase__      :     Optional[Any]=7	,	lowercase__      :     str=3	,	lowercase__      :     Tuple=18	,	lowercase__      :     Dict=30	,	lowercase__      :     List[Any]=400	,	lowercase__      :     Optional[Any]=True	,	lowercase__      :     Union[str, Any]=None	,	lowercase__      :     Union[str, Any]=True	,	lowercase__      :     Tuple=None	,	lowercase__      :     str=True	,	lowercase__      :     List[Any]=[0.48145466, 0.4578275, 0.40821073]	,	lowercase__      :     Optional[Any]=[0.26862954, 0.26130258, 0.27577711]	,	lowercase__      :     str=True	,	):


           '''simple docstring'''

           a_    :       Dict    	=     size if size is not None else {"""shortest_edge""": 20}
           a_    :       Any    	=     crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
           a_    :       List[str]    	=     parent
           a_    :       Dict    	=     batch_size
           a_    :       Any    	=     num_channels
           a_    :       Optional[int]    	=     image_size
           a_    :       Union[str, Any]    	=     min_resolution
           a_    :       Tuple    	=     max_resolution
           a_    :       int    	=     do_resize
           a_    :       List[str]    	=     size
           a_    :       Optional[int]    	=     do_center_crop
           a_    :       str    	=     crop_size
           a_    :       str    	=     do_normalize
           a_    :       Any    	=     image_mean
           a_    :       List[str]    	=     image_std
           a_    :       Any    	=     do_convert_rgb






      def        lowercase_	(   self      :     Optional[int]					):


           '''simple docstring'''

           return {
               "do_resize": self.do_resize,
               "size": self.size,
               "do_center_crop": self.do_center_crop,
               "crop_size": self.crop_size,
               "do_normalize": self.do_normalize,
               "image_mean": self.image_mean,
               "image_std": self.image_std,
               "do_convert_rgb": self.do_convert_rgb,
           }






      def        lowercase_	(   self      :     str	,	lowercase__      :     Union[str, Any]=False	,	lowercase__      :     Union[str, Any]=False	,	lowercase__      :     Union[str, Any]=False					):


           '''simple docstring'''

           assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

           if equal_resolution:
                a_    :       Union[str, Any]    	=     []
                for i in range(self.batch_size					):
                     image_inputs.append(
                         np.random.randint(
                             255	,	size=(self.num_channels, self.max_resolution, self.max_resolution)	,	dtype=np.uinta					)					)
           else:
                a_    :       int    	=     []
                for i in range(self.batch_size					):
                     a_	,    a_    :       Any    	=     np.random.choice(np.arange(self.min_resolution	,	self.max_resolution					)	,	2					)
                     image_inputs.append(np.random.randint(255	,	size=(self.num_channels, width, height)	,	dtype=np.uinta					)					)

           if not numpify and not torchify:
                # PIL expects the channel dimension as last dimension
                a_    :       Union[str, Any]    	=     [Image.fromarray(np.moveaxis(lowercase__	,	0	,	-1					)					) for x in image_inputs]

           if torchify:
                a_    :       List[Any]    	=     [torch.from_numpy(lowercase__					) for x in image_inputs]

           return image_inputs







@require_torch
@require_vision
class     SCREAMING_SNAKE_CASE						(	snake_case_       ,			unittest.TestCase							):
      __magic_name__     :					int             =				CLIPImageProcessor if is_vision_available() else None






      def        lowercase_	(   self      :     List[str]					):


           '''simple docstring'''

           a_    :       Dict    	=     CLIPImageProcessingTester(self					)






      @property
      def        lowercase_	(   self      :     Optional[int]					):


           '''simple docstring'''

           return self.image_processor_tester.prepare_image_processor_dict()






      def        lowercase_	(   self      :     Optional[Any]					):


           '''simple docstring'''

           a_    :       List[Any]    	=     self.image_processing_class(**self.image_processor_dict					)
           self.assertTrue(hasattr(lowercase__	,	"""do_resize"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""size"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_center_crop"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""center_crop"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_normalize"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""image_mean"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""image_std"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_convert_rgb"""					)					)






      def        lowercase_	(   self      :     Dict					):


           '''simple docstring'''

           a_    :       Optional[Any]    	=     self.image_processing_class.from_dict(self.image_processor_dict					)
           self.assertEqual(image_processor.size	,	{"""shortest_edge""": 20}					)
           self.assertEqual(image_processor.crop_size	,	{"""height""": 18, """width""": 18}					)

           a_    :       Optional[int]    	=     self.image_processing_class.from_dict(self.image_processor_dict	,	size=42	,	crop_size=84					)
           self.assertEqual(image_processor.size	,	{"""shortest_edge""": 42}					)
           self.assertEqual(image_processor.crop_size	,	{"""height""": 84, """width""": 84}					)






      def        lowercase_	(   self      :     List[str]					):


           '''simple docstring'''

           pass






      def        lowercase_	(   self      :     Tuple					):


           '''simple docstring'''

           a_    :       str    	=     self.image_processing_class(**self.image_processor_dict					)
           # create random PIL images
           a_    :       str    	=     self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__					)
           for image in image_inputs:
                self.assertIsInstance(lowercase__	,	Image.Image					)

           # Test not batched input
           a_    :       Optional[Any]    	=     image_processing(image_inputs[0]	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)

           # Test batched
           a_    :       str    	=     image_processing(lowercase__	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)






      def        lowercase_	(   self      :     Tuple					):


           '''simple docstring'''

           a_    :       Optional[Any]    	=     self.image_processing_class(**self.image_processor_dict					)
           # create random numpy tensors
           a_    :       Optional[int]    	=     self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__	,	numpify=lowercase__					)
           for image in image_inputs:
                self.assertIsInstance(lowercase__	,	np.ndarray					)

           # Test not batched input
           a_    :       Tuple    	=     image_processing(image_inputs[0]	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)

           # Test batched
           a_    :       str    	=     image_processing(lowercase__	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)






      def        lowercase_	(   self      :     str					):


           '''simple docstring'''

           a_    :       Dict    	=     self.image_processing_class(**self.image_processor_dict					)
           # create random PyTorch tensors
           a_    :       Optional[int]    	=     self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__	,	torchify=lowercase__					)
           for image in image_inputs:
                self.assertIsInstance(lowercase__	,	torch.Tensor					)

           # Test not batched input
           a_    :       Optional[int]    	=     image_processing(image_inputs[0]	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)

           # Test batched
           a_    :       int    	=     image_processing(lowercase__	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)







@require_torch
@require_vision
class     SCREAMING_SNAKE_CASE						(	snake_case_       ,			unittest.TestCase							):
      __magic_name__     :					List[str]             =				CLIPImageProcessor if is_vision_available() else None






      def        lowercase_	(   self      :     str					):


           '''simple docstring'''

           a_    :       str    	=     CLIPImageProcessingTester(self	,	num_channels=4					)
           a_    :       Dict    	=     3






      @property
      def        lowercase_	(   self      :     Any					):


           '''simple docstring'''

           return self.image_processor_tester.prepare_image_processor_dict()






      def        lowercase_	(   self      :     List[Any]					):


           '''simple docstring'''

           a_    :       int    	=     self.image_processing_class(**self.image_processor_dict					)
           self.assertTrue(hasattr(lowercase__	,	"""do_resize"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""size"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_center_crop"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""center_crop"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_normalize"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""image_mean"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""image_std"""					)					)
           self.assertTrue(hasattr(lowercase__	,	"""do_convert_rgb"""					)					)






      def        lowercase_	(   self      :     int					):


           '''simple docstring'''

           pass






      def        lowercase_	(   self      :     Tuple					):


           '''simple docstring'''

           a_    :       List[Any]    	=     self.image_processing_class(**self.image_processor_dict					)
           # create random PIL images
           a_    :       Union[str, Any]    	=     self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__					)
           for image in image_inputs:
                self.assertIsInstance(lowercase__	,	Image.Image					)

           # Test not batched input
           a_    :       Optional[Any]    	=     image_processing(image_inputs[0]	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   1,
                   self.expected_encoded_image_num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)

           # Test batched
           a_    :       List[str]    	=     image_processing(lowercase__	,	return_tensors="""pt"""					).pixel_values
           self.assertEqual(
               encoded_images.shape	,	(
                   self.image_processor_tester.batch_size,
                   self.expected_encoded_image_num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )	,	)
