

"""simple docstring"""

import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_UpperCamelCase		    =						logging.getLogger(__name__)






class  __a				(	__magic_name__					):






       def  SCREAMING_SNAKE_CASE_  (      self ,				snake_case ,				snake_case ,				snake_case=None ,				snake_case=None      ):





        """simple docstring"""

        if current_layer == 0:
         lowerCAmelCase__      :							Tuple    =							self.embedding_hidden_mapping_in(snake_case      )
        else:
         lowerCAmelCase__      :							Any    =							hidden_states[0]

        lowerCAmelCase__      :							Optional[int]    =							int(self.config.num_hidden_layers / self.config.num_hidden_groups      )

        # Index of the hidden group
        lowerCAmelCase__      :							Optional[int]    =							int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)      )

        lowerCAmelCase__      :							int    =							self.albert_layer_groups[group_idx](
            snake_case ,				snake_case ,				head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group] ,				)
        lowerCAmelCase__      :							str    =							layer_group_output[0]

        return (hidden_states,)






@add_start_docstrings(
    'The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'					,  __magic_name__					,  )
class  __a				(	__magic_name__					):






       def __init__(      self ,				snake_case      ):





        """simple docstring"""

        super().__init__(snake_case      )

        lowerCAmelCase__      :							List[Any]    =							AlbertTransformerWithPabee(snake_case      )

        self.init_weights()
        lowerCAmelCase__      :							Union[str, Any]    =							0
        lowerCAmelCase__      :							Optional[Any]    =							0
        lowerCAmelCase__      :							str    =							0

        lowerCAmelCase__      :							Any    =							0






       def  SCREAMING_SNAKE_CASE_  (      self ,				snake_case      ):





        """simple docstring"""

        lowerCAmelCase__      :							Union[str, Any]    =							threshold






       def  SCREAMING_SNAKE_CASE_  (      self ,				snake_case      ):





        """simple docstring"""

        lowerCAmelCase__      :							Optional[int]    =							patience






       def  SCREAMING_SNAKE_CASE_  (      self      ):





        """simple docstring"""

        lowerCAmelCase__      :							Union[str, Any]    =							0
        lowerCAmelCase__      :							str    =							0






       def  SCREAMING_SNAKE_CASE_  (      self      ):





        """simple docstring"""

        lowerCAmelCase__      :							List[str]    =							self.inference_layers_num / self.inference_instances_num
        lowerCAmelCase__      :							Optional[int]    =							(
            F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
            F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
        )
        print(snake_case      )






       @add_start_docstrings_to_model_forward(snake_case      )
       def  SCREAMING_SNAKE_CASE_  (      self ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=False ,				):





        """simple docstring"""

        if input_ids is not None and inputs_embeds is not None:
         raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"      )
        elif input_ids is not None:
         lowerCAmelCase__      :							List[str]    =							input_ids.size()
        elif inputs_embeds is not None:
         lowerCAmelCase__      :							str    =							inputs_embeds.size()[:-1]
        else:
         raise ValueError("You have to specify either input_ids or inputs_embeds"      )

        lowerCAmelCase__      :							Union[str, Any]    =							input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
         lowerCAmelCase__      :							Union[str, Any]    =							torch.ones(snake_case ,				device=snake_case      )
        if token_type_ids is None:
         lowerCAmelCase__      :							Optional[int]    =							torch.zeros(snake_case ,				dtype=torch.long ,				device=snake_case      )

        lowerCAmelCase__      :							Tuple    =							attention_mask.unsqueeze(1      ).unsqueeze(2      )
        lowerCAmelCase__      :							Optional[Any]    =							extended_attention_mask.to(dtype=self.dtype      )  # fp16 compatibility
        lowerCAmelCase__      :							int    =							(1.0 - extended_attention_mask) * -10_000.0
        lowerCAmelCase__      :							Tuple    =							self.get_head_mask(snake_case ,				self.config.num_hidden_layers      )

        lowerCAmelCase__      :							Dict    =							self.embeddings(
            snake_case ,				position_ids=snake_case ,				token_type_ids=snake_case ,				inputs_embeds=snake_case      )
        lowerCAmelCase__      :							str    =							embedding_output

        if self.training:
         lowerCAmelCase__      :							Union[str, Any]    =							[]
         for i in range(self.config.num_hidden_layers      ):
          lowerCAmelCase__      :							Dict    =							self.encoder.adaptive_forward(
              snake_case ,				current_layer=snake_case ,				attention_mask=snake_case ,				head_mask=snake_case ,				)

          lowerCAmelCase__      :							Any    =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
          lowerCAmelCase__      :							Dict    =							output_layers[i](output_dropout(snake_case      )      )
          res.append(snake_case      )
        elif self.patience == 0:  # Use all layers for inference
         lowerCAmelCase__      :							Tuple    =							self.encoder(snake_case ,				snake_case ,				head_mask=snake_case      )
         lowerCAmelCase__      :							str    =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
         lowerCAmelCase__      :							Optional[Any]    =							[output_layers[self.config.num_hidden_layers - 1](snake_case      )]
        else:
         lowerCAmelCase__      :							Dict    =							0
         lowerCAmelCase__      :							Tuple    =							None
         lowerCAmelCase__      :							List[str]    =							0
         for i in range(self.config.num_hidden_layers      ):
          calculated_layer_num += 1
          lowerCAmelCase__      :							Optional[Any]    =							self.encoder.adaptive_forward(
              snake_case ,				current_layer=snake_case ,				attention_mask=snake_case ,				head_mask=snake_case ,				)

          lowerCAmelCase__      :							Union[str, Any]    =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
          lowerCAmelCase__      :							Any    =							output_layers[i](snake_case      )
          if regression:
           lowerCAmelCase__      :							Tuple    =							logits.detach()
           if patient_result is not None:
            lowerCAmelCase__      :							Tuple    =							patient_result.detach()
           if (patient_result is not None) and torch.abs(patient_result - labels      ) < self.regression_threshold:
            patient_counter += 1
           else:
            lowerCAmelCase__      :							str    =							0
          else:
           lowerCAmelCase__      :							int    =							logits.detach().argmax(dim=1      )
           if patient_result is not None:
            lowerCAmelCase__      :							Optional[Any]    =							patient_result.detach().argmax(dim=1      )
           if (patient_result is not None) and torch.all(labels.eq(snake_case      )      ):
            patient_counter += 1
           else:
            lowerCAmelCase__      :							Union[str, Any]    =							0

          lowerCAmelCase__      :							Any    =							logits
          if patient_counter == self.patience:
           break
         lowerCAmelCase__      :							Optional[Any]    =							[patient_result]
         self.inference_layers_num += calculated_layer_num
         self.inference_instances_num += 1

        return res






@add_start_docstrings(
    'Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '					,  __magic_name__					,  )
class  __a				(	__magic_name__					):






       def __init__(      self ,				snake_case      ):





        """simple docstring"""

        super().__init__(snake_case      )
        lowerCAmelCase__      :							List[str]    =							config.num_labels

        lowerCAmelCase__      :							Any    =							AlbertModelWithPabee(snake_case      )
        lowerCAmelCase__      :							Optional[Any]    =							nn.Dropout(config.classifier_dropout_prob      )
        lowerCAmelCase__      :							List[Any]    =							nn.ModuleList(
            [nn.Linear(config.hidden_size ,				self.config.num_labels      ) for _ in range(config.num_hidden_layers      )]      )

        self.init_weights()






       @add_start_docstrings_to_model_forward(snake_case      )
       def  SCREAMING_SNAKE_CASE_  (      self ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				snake_case=None ,				):





        """simple docstring"""

        lowerCAmelCase__      :							List[str]    =							self.albert(
            input_ids=snake_case ,				attention_mask=snake_case ,				token_type_ids=snake_case ,				position_ids=snake_case ,				head_mask=snake_case ,				inputs_embeds=snake_case ,				output_dropout=self.dropout ,				output_layers=self.classifiers ,				regression=self.num_labels == 1 ,				)

        lowerCAmelCase__      :							int    =							(logits[-1],)

        if labels is not None:
         lowerCAmelCase__      :							Tuple    =							None
         lowerCAmelCase__      :							str    =							0
         for ix, logits_item in enumerate(snake_case      ):
          if self.num_labels == 1:
           #  We are doing regression
           lowerCAmelCase__      :							str    =							MSELoss()
           lowerCAmelCase__      :							int    =							loss_fct(logits_item.view(-1      ) ,				labels.view(-1      )      )
          else:
           lowerCAmelCase__      :							List[str]    =							CrossEntropyLoss()
           lowerCAmelCase__      :							Union[str, Any]    =							loss_fct(logits_item.view(-1 ,				self.num_labels      ) ,				labels.view(-1      )      )
          if total_loss is None:
           lowerCAmelCase__      :							Any    =							loss
          else:
           total_loss += loss * (ix + 1)
          total_weights += ix + 1
         lowerCAmelCase__      :							str    =							(total_loss / total_weights,) + outputs

        return outputs









"""simple docstring"""

import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


_UpperCamelCase		    =						logging.getLogger(__name__)






@dataclass
class  __a				:





       """simple docstring"""


       __UpperCamelCase				:			str          =	field(
           metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}					)
       __UpperCamelCase				:			Optional[str]          =	field(
           default=__magic_name__					,  metadata={'help': 'Pretrained config name or path if not the same as model_name'}					)
       __UpperCamelCase				:			Optional[str]          =	field(
           default=__magic_name__					,  metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}					)
       __UpperCamelCase				:			Optional[str]          =	field(
           default=__magic_name__					,  metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}					,  )






@dataclass
class  __a				:





       """simple docstring"""


       __UpperCamelCase				:			str          =	field(
           metadata={'help': 'The name of the task to train selected in the list: ' + ', '.join(hans_processors.keys()					)}					)
       __UpperCamelCase				:			str          =	field(
           metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}					)
       __UpperCamelCase				:			int          =	field(
           default=128					,  metadata={
               'help': (
                   'The maximum total input sequence length after tokenization. Sequences longer '
                   'than this will be truncated, sequences shorter will be padded.'
               )
           }					,  )
       __UpperCamelCase				:			bool          =	field(
           default=__magic_name__					,  metadata={'help': 'Overwrite the cached training and evaluation sets'}					)




def 			SCREAMING_SNAKE_CASE       (   lowercase__      ) ->	Dict[str, torch.Tensor]:
 lowerCAmelCase__      :							List[Any]    =							default_data_collator(lowercase__      )
 lowerCAmelCase__      :							Union[str, Any]    =							batch.pop("pairID"				,				lowercase__      )
 return batch



def 			SCREAMING_SNAKE_CASE       (   ) ->	int:
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.

 lowerCAmelCase__      :							str    =							HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)      )
 lowerCAmelCase__ ,	lowerCAmelCase__ ,	lowerCAmelCase__      :							List[Any]    =							parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir      )
     and os.listdir(training_args.output_dir      )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
      " --overwrite_output_dir to overcome."      )

 # Setup logging
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"				,				datefmt="%m/%d/%Y %H:%M:%S"				,				level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN				,				)
 logger.warning(
     "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"				,				training_args.local_rank				,				training_args.device				,				training_args.n_gpu				,				bool(training_args.local_rank != -1      )				,				training_args.fpaa				,				)
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank      ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info("Training/evaluation parameters %s"				,				lowercase__      )

 # Set seed
 set_seed(training_args.seed      )

 try:
  lowerCAmelCase__      :							Tuple    =							hans_tasks_num_labels[data_args.task_name]
 except KeyError:
  raise ValueError("Task not found: %s" % (data_args.task_name)      )

 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 lowerCAmelCase__      :							int    =							AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path				,				num_labels=lowercase__				,				finetuning_task=data_args.task_name				,				cache_dir=model_args.cache_dir				,				)
 lowerCAmelCase__      :							Optional[int]    =							AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path				,				cache_dir=model_args.cache_dir				,				)
 lowerCAmelCase__      :							Union[str, Any]    =							AutoModelForSequenceClassification.from_pretrained(
     model_args.model_name_or_path				,				from_tf=bool(".ckpt" in model_args.model_name_or_path      )				,				config=lowercase__				,				cache_dir=model_args.cache_dir				,				)

 # Get datasets
 lowerCAmelCase__      :							Dict    =							(
     HansDataset(
         data_dir=data_args.data_dir				,				tokenizer=lowercase__				,				task=data_args.task_name				,				max_seq_length=data_args.max_seq_length				,				overwrite_cache=data_args.overwrite_cache				,				)
     if training_args.do_train
     else None
 )
 lowerCAmelCase__      :							Union[str, Any]    =							(
     HansDataset(
         data_dir=data_args.data_dir				,				tokenizer=lowercase__				,				task=data_args.task_name				,				max_seq_length=data_args.max_seq_length				,				overwrite_cache=data_args.overwrite_cache				,				evaluate=lowercase__				,				)
     if training_args.do_eval
     else None
 )

 # Initialize our Trainer
 lowerCAmelCase__      :							Any    =							Trainer(
     model=lowercase__				,				args=lowercase__				,				train_dataset=lowercase__				,				eval_dataset=lowercase__				,				data_collator=lowercase__				,				)

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path      ) else None      )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir      )

    # Evaluation
 if training_args.do_eval:
  logger.info("*** Evaluate ***"      )

  lowerCAmelCase__      :							Tuple    =							trainer.predict(lowercase__      )
  lowerCAmelCase__      :							Optional[Any]    =							output.predictions
  lowerCAmelCase__      :							Dict    =							np.argmax(lowercase__				,				axis=1      )

  lowerCAmelCase__      :							int    =							[ex.pairID for ex in eval_dataset]
  lowerCAmelCase__      :							Tuple    =							os.path.join(training_args.output_dir				,				"hans_predictions.txt"      )
  lowerCAmelCase__      :							Tuple    =							eval_dataset.get_labels()
  if trainer.is_world_master():
   with open(lowercase__				,				"w"      ) as writer:
    writer.write("pairID,gold_label\n"      )
    for pid, pred in zip(lowercase__				,				lowercase__      ):
     writer.write("ex" + str(lowercase__      ) + "," + label_list[int(lowercase__      )] + "\n"      )

  trainer._log(output.metrics      )



def 			SCREAMING_SNAKE_CASE       (   lowercase__      ) ->	Optional[Any]:
 # For xla_spawn (TPUs)
 main()


if __name__ == "__main__":
    main()






