





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


SCREAMING_SNAKE_CASE_      :			Optional[int]		  =      logging.getLogger(__name__)

class 		snake_case_      (       UpperCAmelCase_	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    Union[str, Any]				,			__lowerCamelCase       :    List[Any]				,			__lowerCamelCase       :    List[Any]				,			__lowerCamelCase       :    str=None				,			__lowerCamelCase       :    Optional[int]=None      )					->   str:

			'''simple docstring'''




			if current_layer == 0:
					__lowercase					=    self.embedding_hidden_mapping_in(__lowerCamelCase      )
			else:
					__lowercase					=    hidden_states[0]

			__lowercase					=    int(self.config.num_hidden_layers / self.config.num_hidden_groups      )

			# Index of the hidden group
			__lowercase					=    int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)      )

			__lowercase					=    self.albert_layer_groups[group_idx](
			    __lowerCamelCase				,			__lowerCamelCase				,			head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]				,			)
			__lowercase					=    layer_group_output[0]

			return (hidden_states,)





@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''    ,    UpperCAmelCase_    ,    )
class 		snake_case_      (       UpperCAmelCase_	):







	'''simple docstring'''




	def __init__(	self       :    Dict				,			__lowerCamelCase       :    List[Any]      )					->   str:

			'''simple docstring'''




			super().__init__(__lowerCamelCase      )

			__lowercase					=    AlbertTransformerWithPabee(__lowerCamelCase      )

			self.init_weights()
			__lowercase					=    0
			__lowercase					=    0
			__lowercase					=    0

			__lowercase					=    0




	def       UpperCAmelCase (	self       :    List[Any]				,			__lowerCamelCase       :    Optional[Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    threshold




	def       UpperCAmelCase (	self       :    List[Any]				,			__lowerCamelCase       :    Optional[Any]      )					->   Dict:

			'''simple docstring'''




			__lowercase					=    patience




	def       UpperCAmelCase (	self       :    List[Any]      )					->   List[str]:

			'''simple docstring'''




			__lowercase					=    0
			__lowercase					=    0




	def       UpperCAmelCase (	self       :    List[str]      )					->   List[Any]:

			'''simple docstring'''




			__lowercase					=    self.inference_layers_num / self.inference_instances_num
			__lowercase					=    (
			    F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
			    F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
			)
			print(__lowerCamelCase      )






	@add_start_docstrings_to_model_forward(__lowerCamelCase      )
	def       UpperCAmelCase (	self       :    Dict				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    Optional[int]=None				,			__lowerCamelCase       :    Union[str, Any]=None				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    Any=None				,			__lowerCamelCase       :    Union[str, Any]=None				,			__lowerCamelCase       :    List[Any]=False				,			)					->   int:

			'''simple docstring'''




			if input_ids is not None and inputs_embeds is not None:
					raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'      )
			elif input_ids is not None:
					__lowercase					=    input_ids.size()
			elif inputs_embeds is not None:
					__lowercase					=    inputs_embeds.size()[:-1]
			else:
					raise ValueError('You have to specify either input_ids or inputs_embeds'      )

			__lowercase					=    input_ids.device if input_ids is not None else inputs_embeds.device

			if attention_mask is None:
					__lowercase					=    torch.ones(__lowerCamelCase				,			device=__lowerCamelCase      )
			if token_type_ids is None:
					__lowercase					=    torch.zeros(__lowerCamelCase				,			dtype=torch.long				,			device=__lowerCamelCase      )

			__lowercase					=    attention_mask.unsqueeze(1      ).unsqueeze(2      )
			__lowercase					=    extended_attention_mask.to(dtype=self.dtype      )  # fp16 compatibility
			__lowercase					=    (1.0 - extended_attention_mask) * -10000.0
			__lowercase					=    self.get_head_mask(__lowerCamelCase				,			self.config.num_hidden_layers      )

			__lowercase					=    self.embeddings(
			    __lowerCamelCase				,			position_ids=__lowerCamelCase				,			token_type_ids=__lowerCamelCase				,			inputs_embeds=__lowerCamelCase      )
			__lowercase					=    embedding_output

			if self.training:
					__lowercase					=    []
					for i in range(self.config.num_hidden_layers      ):
							__lowercase					=    self.encoder.adaptive_forward(
							    __lowerCamelCase				,			current_layer=__lowerCamelCase				,			attention_mask=__lowerCamelCase				,			head_mask=__lowerCamelCase				,			)

							__lowercase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
							__lowercase					=    output_layers[i](output_dropout(__lowerCamelCase      )      )
							res.append(__lowerCamelCase      )
			elif self.patience == 0:  # Use all layers for inference
					__lowercase					=    self.encoder(__lowerCamelCase				,			__lowerCamelCase				,			head_mask=__lowerCamelCase      )
					__lowercase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
					__lowercase					=    [output_layers[self.config.num_hidden_layers - 1](__lowerCamelCase      )]
			else:
					__lowercase					=    0
					__lowercase					=    None
					__lowercase					=    0
					for i in range(self.config.num_hidden_layers      ):
							calculated_layer_num += 1
							__lowercase					=    self.encoder.adaptive_forward(
							    __lowerCamelCase				,			current_layer=__lowerCamelCase				,			attention_mask=__lowerCamelCase				,			head_mask=__lowerCamelCase				,			)

							__lowercase					=    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
							__lowercase					=    output_layers[i](__lowerCamelCase      )
							if regression:
									__lowercase					=    logits.detach()
									if patient_result is not None:
											__lowercase					=    patient_result.detach()
									if (patient_result is not None) and torch.abs(patient_result - labels      ) < self.regression_threshold:
											patient_counter += 1
									else:
											__lowercase					=    0
							else:
									__lowercase					=    logits.detach().argmax(dim=1      )
									if patient_result is not None:
											__lowercase					=    patient_result.detach().argmax(dim=1      )
									if (patient_result is not None) and torch.all(labels.eq(__lowerCamelCase      )      ):
											patient_counter += 1
									else:
											__lowercase					=    0

							__lowercase					=    logits
							if patient_counter == self.patience:
									break
					__lowercase					=    [patient_result]
					self.inference_layers_num += calculated_layer_num
					self.inference_instances_num += 1

			return res





@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''    ,    UpperCAmelCase_    ,    )
class 		snake_case_      (       UpperCAmelCase_	):







	'''simple docstring'''




	def __init__(	self       :    Optional[int]				,			__lowerCamelCase       :    int      )					->   int:

			'''simple docstring'''




			super().__init__(__lowerCamelCase      )
			__lowercase					=    config.num_labels

			__lowercase					=    AlbertModelWithPabee(__lowerCamelCase      )
			__lowercase					=    nn.Dropout(config.classifier_dropout_prob      )
			__lowercase					=    nn.ModuleList(
			    [nn.Linear(config.hidden_size				,			self.config.num_labels      ) for _ in range(config.num_hidden_layers      )]      )

			self.init_weights()






	@add_start_docstrings_to_model_forward(__lowerCamelCase      )
	def       UpperCAmelCase (	self       :    Optional[Any]				,			__lowerCamelCase       :    Optional[int]=None				,			__lowerCamelCase       :    int=None				,			__lowerCamelCase       :    List[Any]=None				,			__lowerCamelCase       :    Tuple=None				,			__lowerCamelCase       :    Tuple=None				,			__lowerCamelCase       :    str=None				,			__lowerCamelCase       :    Optional[Any]=None				,			)					->   Dict:

			'''simple docstring'''




			__lowercase					=    self.albert(
			    input_ids=__lowerCamelCase				,			attention_mask=__lowerCamelCase				,			token_type_ids=__lowerCamelCase				,			position_ids=__lowerCamelCase				,			head_mask=__lowerCamelCase				,			inputs_embeds=__lowerCamelCase				,			output_dropout=self.dropout				,			output_layers=self.classifiers				,			regression=self.num_labels == 1				,			)

			__lowercase					=    (logits[-1],)

			if labels is not None:
					__lowercase					=    None
					__lowercase					=    0
					for ix, logits_item in enumerate(__lowerCamelCase      ):
							if self.num_labels == 1:
									#  We are doing regression
									__lowercase					=    MSELoss()
									__lowercase					=    loss_fct(logits_item.view(-1      )				,			labels.view(-1      )      )
							else:
									__lowercase					=    CrossEntropyLoss()
									__lowercase					=    loss_fct(logits_item.view(-1				,			self.num_labels      )				,			labels.view(-1      )      )
							if total_loss is None:
									__lowercase					=    loss
							else:
									total_loss += loss * (ix + 1)
							total_weights += ix + 1
					__lowercase					=    (total_loss / total_weights,) + outputs

			return outputs











# Usage:
# ./gen-card-allenai-wmt19.py

import os
from pathlib import Path



def    SCREAMING_SNAKE_CASE							(			snake_case	,      snake_case	,      snake_case	,      snake_case       )		->						Optional[Any]:
		__lowercase					=    {
		    'en': 'Machine learning is great, isn\'t it?',
		    'ru': 'Машинное обучение - это здорово, не так ли?',
		    'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
		}

		# BLUE scores as follows:
		# "pair": [fairseq, transformers]
		__lowercase					=    {
		    'wmt19-de-en-6-6-base': [0, 38.37],
		    'wmt19-de-en-6-6-big': [0, 39.90],
		}
		__lowercase					=    F"{src_lang}-{tgt_lang}"

		__lowercase					=    F"\n---\n\nlanguage:\n- {src_lang}\n- {tgt_lang}\nthumbnail:\ntags:\n- translation\n- wmt19\n- allenai\nlicense: apache-2.0\ndatasets:\n- wmt19\nmetrics:\n- bleu\n---\n\n# FSMT\n\n## Model description\n\nThis is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.\n\nFor more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).\n\n2 models are available:\n\n* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)\n* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)\n\n\n## Intended uses & limitations\n\n#### How to use\n\n```python\nfrom transformers import FSMTForConditionalGeneration, FSMTTokenizer\nmname = \"allenai/{model_name}\"\ntokenizer = FSMTTokenizer.from_pretrained(mname)\nmodel = FSMTForConditionalGeneration.from_pretrained(mname)\n\ninput = \"{texts[src_lang]}\"\ninput_ids = tokenizer.encode(input, return_tensors=\"pt\")\noutputs = model.generate(input_ids)\ndecoded = tokenizer.decode(outputs[0], skip_special_tokens=True)\nprint(decoded) # {texts[tgt_lang]}\n\n```\n\n#### Limitations and bias\n\n\n## Training data\n\nPretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).\n\n## Eval results\n\nHere are the BLEU scores:\n\nmodel   |  transformers\n-------|---------\n{model_name}  |  {scores[model_name][1]}\n\nThe score was calculated using this code:\n\n```bash\ngit clone https://github.com/huggingface/transformers\ncd transformers\nexport PAIR={pair}\nexport DATA_DIR=data/$PAIR\nexport SAVE_DIR=data/$PAIR\nexport BS=8\nexport NUM_BEAMS=5\nmkdir -p $DATA_DIR\nsacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source\nsacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target\necho $PAIR\nPYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS\n```\n\n## Data Sources\n\n- [training, etc.](http://www.statmt.org/wmt19/)\n- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)\n\n\n### BibTeX entry and citation info\n\n```\n@misc{{kasai2020deep,\n    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},\n    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},\n    year={{2020}},\n    eprint={{2006.10369}},\n    archivePrefix={{arXiv}},\n    primaryClass={{cs.CL}}\n}}\n```\n\n"
		model_card_dir.mkdir(parents=snake_case	,      exist_ok=snake_case       )
		__lowercase					=    os.path.join(snake_case	,      'README.md'       )
		print(F"Generating {path}"       )
		with open(snake_case	,      'w'	,      encoding='utf-8'       ) as f:
				f.write(snake_case       )


# make sure we are under the root of the project
SCREAMING_SNAKE_CASE_      :			Tuple		  =      Path(__file__).resolve().parent.parent.parent
SCREAMING_SNAKE_CASE_      :			Any		  =      repo_dir / '''model_cards'''

for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
		SCREAMING_SNAKE_CASE_      :			Any		  =      model_cards_dir / '''allenai''' / model_name
		write_model_card(model_card_dir, src_lang='''de''', tgt_lang='''en''', model_name=model_name)




