





# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List



def    SCREAMING_SNAKE_CASE							(			snake_case	,      snake_case       )		->						Dict:
		if name != "training_function" and name != "main":
				raise ValueError(F"Incorrect function name passed: {name}, choose either 'main' or 'training_function'"       )
		__lowercase	,			__lowercase					=    [], False
		for line in lines:
				if not found_start and F"def {name}" in line:
						__lowercase					=    True
						good_lines.append(snake_case       )
						continue
				if found_start:
						if name == "training_function" and "def main" in line:
								return good_lines
						if name == "main" and "if __name__" in line:
								return good_lines
						good_lines.append(snake_case       )



def    SCREAMING_SNAKE_CASE							(			snake_case       )		->						Any:
		return [line for line in lines if not line.lstrip().startswith('#'       ) and line != "\n"]



def    SCREAMING_SNAKE_CASE							(			snake_case	,      snake_case	,      snake_case	,      snake_case = None       )		->						Dict:
		with open(snake_case	,      'r'       ) as f:
				__lowercase					=    f.readlines()
		with open(os.path.abspath(os.path.join('examples'	,      'nlp_example.py'       )       )	,      'r'       ) as f:
				__lowercase					=    f.readlines()
		with open(snake_case	,      'r'       ) as f:
				__lowercase					=    f.readlines()
		if secondary_filename is not None:
				with open(snake_case	,      'r'       ) as f:
						__lowercase					=    f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
		if parser_only:
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'main'       )       )
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'main'       )       )
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'main'       )       )
				if secondary_filename is not None:
						__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'main'       )       )
		else:
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'training_function'       )       )
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'training_function'       )       )
				__lowercase					=    clean_lines(get_function_contents_by_name(snake_case	,      'training_function'       )       )
				if secondary_filename is not None:
						__lowercase					=    clean_lines(
						    get_function_contents_by_name(snake_case	,      'training_function'       )       )

		__lowercase					=    'train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'

		# Specific code in our script that differs from the full version, aka what is new
		__lowercase					=    []
		__lowercase					=    []  # We keep track of the idxs just in case it's a repeated statement
		__lowercase					=    iter(snake_case       )
		for i in range(len(snake_case       ) - 1       ):
				if i not in passed_idxs:
						__lowercase					=    next(snake_case       )
						if (line not in full_file_func) and (line.lstrip() != _dl_line):
								if "TESTING_MOCKED_DATALOADERS" not in line:
										new_feature_code.append(snake_case       )
										passed_idxs.append(snake_case       )
								else:
										# Skip over the `config['num_epochs'] = 2` statement
										__lowercase					=    next(snake_case       )

    # Extract out just the new parts from the full_file_training_func
		__lowercase					=    []
		__lowercase					=    []  # We keep track of the idxs just in case it's a repeated statement
		for i, line in enumerate(snake_case       ):
				if i not in passed_idxs:
						if (line not in full_file_func) and (line.lstrip() != _dl_line):
								if "TESTING_MOCKED_DATALOADERS" not in line:
										new_full_example_parts.append(snake_case       )
										passed_idxs.append(snake_case       )

    # Finally, get the overall diff
		__lowercase					=    [line for line in new_feature_code if line not in new_full_example_parts]
		if secondary_filename is not None:
				__lowercase					=    [line for line in full_file_contents if line not in secondary_file_func]
				__lowercase					=    [line for line in diff_from_example if line not in diff_from_two]

		return diff_from_example











# Usage:
# ./gen-card-allenai-wmt19.py

import os
from pathlib import Path



def    SCREAMING_SNAKE_CASE							(			snake_case	,      snake_case	,      snake_case	,      snake_case       )		->						Optional[Any]:
		__lowercase					=    {
		    'en': 'Machine learning is great, isn\'t it?',
		    'ru': 'Машинное обучение - это здорово, не так ли?',
		    'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
		}

		# BLUE scores as follows:
		# "pair": [fairseq, transformers]
		__lowercase					=    {
		    'wmt19-de-en-6-6-base': [0, 38.37],
		    'wmt19-de-en-6-6-big': [0, 39.90],
		}
		__lowercase					=    F"{src_lang}-{tgt_lang}"

		__lowercase					=    F"\n---\n\nlanguage:\n- {src_lang}\n- {tgt_lang}\nthumbnail:\ntags:\n- translation\n- wmt19\n- allenai\nlicense: apache-2.0\ndatasets:\n- wmt19\nmetrics:\n- bleu\n---\n\n# FSMT\n\n## Model description\n\nThis is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.\n\nFor more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).\n\n2 models are available:\n\n* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)\n* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)\n\n\n## Intended uses & limitations\n\n#### How to use\n\n```python\nfrom transformers import FSMTForConditionalGeneration, FSMTTokenizer\nmname = \"allenai/{model_name}\"\ntokenizer = FSMTTokenizer.from_pretrained(mname)\nmodel = FSMTForConditionalGeneration.from_pretrained(mname)\n\ninput = \"{texts[src_lang]}\"\ninput_ids = tokenizer.encode(input, return_tensors=\"pt\")\noutputs = model.generate(input_ids)\ndecoded = tokenizer.decode(outputs[0], skip_special_tokens=True)\nprint(decoded) # {texts[tgt_lang]}\n\n```\n\n#### Limitations and bias\n\n\n## Training data\n\nPretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).\n\n## Eval results\n\nHere are the BLEU scores:\n\nmodel   |  transformers\n-------|---------\n{model_name}  |  {scores[model_name][1]}\n\nThe score was calculated using this code:\n\n```bash\ngit clone https://github.com/huggingface/transformers\ncd transformers\nexport PAIR={pair}\nexport DATA_DIR=data/$PAIR\nexport SAVE_DIR=data/$PAIR\nexport BS=8\nexport NUM_BEAMS=5\nmkdir -p $DATA_DIR\nsacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source\nsacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target\necho $PAIR\nPYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS\n```\n\n## Data Sources\n\n- [training, etc.](http://www.statmt.org/wmt19/)\n- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)\n\n\n### BibTeX entry and citation info\n\n```\n@misc{{kasai2020deep,\n    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},\n    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},\n    year={{2020}},\n    eprint={{2006.10369}},\n    archivePrefix={{arXiv}},\n    primaryClass={{cs.CL}}\n}}\n```\n\n"
		model_card_dir.mkdir(parents=snake_case	,      exist_ok=snake_case       )
		__lowercase					=    os.path.join(snake_case	,      'README.md'       )
		print(F"Generating {path}"       )
		with open(snake_case	,      'w'	,      encoding='utf-8'       ) as f:
				f.write(snake_case       )


# make sure we are under the root of the project
SCREAMING_SNAKE_CASE_      :			Tuple		  =      Path(__file__).resolve().parent.parent.parent
SCREAMING_SNAKE_CASE_      :			Any		  =      repo_dir / '''model_cards'''

for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
		SCREAMING_SNAKE_CASE_      :			Any		  =      model_cards_dir / '''allenai''' / model_name
		write_model_card(model_card_dir, src_lang='''de''', tgt_lang='''en''', model_name=model_name)




