



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 			_lowercase							(     A_,   unittest.TestCase			):





	"""simple docstring"""
	__A					=							ElectraTokenizer
	__A					=							ElectraTokenizerFast
	__A					=							True
	__A					=							True
	__A					=							filter_non_english




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						super().setUp()

						a        =     [
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "[PAD]",
						    "[MASK]",
						    "want",
						    "##want",
						    "##ed",
						    "wa",
						    "un",
						    "runn",
						    "##ing",
						    ",",
						    "low",
						    "lowest",
						]
						a        =     os.path.join(self.tmpdirname	, VOCAB_FILES_NAMES["vocab_file"]	)
						with open(self.vocab_file	, "w"	, encoding="utf-8"	) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)




	def   UpperCamelCase_ (self	, lowerCamelCase_	):
						"""simple docstring"""

						a        =     "UNwant\u00E9d,running"
						a        =     "unwanted, running"
						return input_text, output_text




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     self.tokenizer_class(self.vocab_file	)

						a        =     tokenizer.tokenize("UNwant\u00E9d,running"	)
						self.assertListEqual(lowerCamelCase_	, ["un", "##want", "##ed", ",", "runn", "##ing"]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_	)	, [9, 6, 7, 12, 10, 11]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						if not self.test_rust_tokenizer:
											return

						a        =     self.get_tokenizer()
						a        =     self.get_rust_tokenizer()

						a        =     "UNwant\u00E9d,running"

						a        =     tokenizer.tokenize(lowerCamelCase_	)
						a        =     rust_tokenizer.tokenize(lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

						a        =     tokenizer.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
						a        =     rust_tokenizer.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

						a        =     self.get_rust_tokenizer()
						a        =     tokenizer.encode(lowerCamelCase_	)
						a        =     rust_tokenizer.encode(lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

						# With lower casing
						a        =     self.get_tokenizer(do_lower_case=lowerCamelCase_	)
						a        =     self.get_rust_tokenizer(do_lower_case=lowerCamelCase_	)

						a        =     "UNwant\u00E9d,running"

						a        =     tokenizer.tokenize(lowerCamelCase_	)
						a        =     rust_tokenizer.tokenize(lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

						a        =     tokenizer.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
						a        =     rust_tokenizer.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

						a        =     self.get_rust_tokenizer()
						a        =     tokenizer.encode(lowerCamelCase_	)
						a        =     rust_tokenizer.encode(lowerCamelCase_	)
						self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)	, ["ah", "\u535A", "\u63A8", "zz"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)	, ["hello", "!", "how", "are", "you", "?"]	)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)	, ["hello"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	, strip_accents=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)	, ["hällo", "!", "how", "are", "you", "?"]	)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)	, ["h\u00E9llo"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	, strip_accents=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)	, ["hallo", "!", "how", "are", "you", "?"]	)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)	, ["hello"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)	, ["hallo", "!", "how", "are", "you", "?"]	)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)	, ["hello"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)	, ["HeLLo", "!", "how", "Are", "yoU", "?"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	, strip_accents=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)	, ["HäLLo", "!", "how", "Are", "yoU", "?"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	, strip_accents=lowerCamelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)	, ["HaLLo", "!", "how", "Are", "yoU", "?"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     BasicTokenizer(do_lower_case=lowerCamelCase_	, never_split=["[UNK]"]	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)	, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						a        =     {}
						for i, token in enumerate(lowerCamelCase_	):
											a        =     i
						a        =     WordpieceTokenizer(vocab=lowerCamelCase_	, unk_token="[UNK]"	)

						self.assertListEqual(tokenizer.tokenize(""	)	, []	)

						self.assertListEqual(tokenizer.tokenize("unwanted running"	)	, ["un", "##want", "##ed", "runn", "##ing"]	)

						self.assertListEqual(tokenizer.tokenize("unwantedX running"	)	, ["[UNK]", "runn", "##ing"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						self.assertTrue(_is_whitespace(" "	)	)
						self.assertTrue(_is_whitespace("\t"	)	)
						self.assertTrue(_is_whitespace("\r"	)	)
						self.assertTrue(_is_whitespace("\n"	)	)
						self.assertTrue(_is_whitespace("\u00A0"	)	)

						self.assertFalse(_is_whitespace("A"	)	)
						self.assertFalse(_is_whitespace("-"	)	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						self.assertTrue(_is_control("\u0005"	)	)

						self.assertFalse(_is_control("A"	)	)
						self.assertFalse(_is_control(" "	)	)
						self.assertFalse(_is_control("\t"	)	)
						self.assertFalse(_is_control("\r"	)	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						self.assertTrue(_is_punctuation("-"	)	)
						self.assertTrue(_is_punctuation("$"	)	)
						self.assertTrue(_is_punctuation("`"	)	)
						self.assertTrue(_is_punctuation("."	)	)

						self.assertFalse(_is_punctuation("A"	)	)
						self.assertFalse(_is_punctuation(" "	)	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     self.get_tokenizer()
						a        =     self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(lowerCamelCase_	) for t in ["Test", "\xad", "test"]]	, [["[UNK]"], [], ["[UNK]"]]	)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(lowerCamelCase_	) for t in ["Test", "\xad", "test"]]	, [["[UNK]"], [], ["[UNK]"]]	)




	@slow
	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)

						a        =     tokenizer.encode("sequence builders"	, add_special_tokens=lowerCamelCase_	)
						a        =     tokenizer.encode("multi-sequence build"	, add_special_tokens=lowerCamelCase_	)

						a        =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_	)
						a        =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_	, lowerCamelCase_	)

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																a        =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)

																a        =     F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																a        =     tokenizer_r.encode_plus(
																    lowerCamelCase_	, return_attention_mask=lowerCamelCase_	, return_token_type_ids=lowerCamelCase_	, return_offsets_mapping=lowerCamelCase_	, add_special_tokens=lowerCamelCase_	, )

																a        =     tokenizer_r.do_lower_case if hasattr(lowerCamelCase_	, "do_lower_case"	) else False
																a        =     (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "A"),
																        ((1, 2), ","),
																        ((3, 5), "na"),
																        ((5, 6), "##ï"),
																        ((6, 8), "##ve"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "Allen"),
																        ((21, 23), "##NL"),
																        ((23, 24), "##P"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "a"),
																        ((1, 2), ","),
																        ((3, 8), "naive"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "allen"),
																        ((21, 23), "##nl"),
																        ((23, 24), "##p"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]	, tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
																self.assertEqual([e[0] for e in expected_results]	, tokens["offset_mapping"]	)




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     ["的", "人", "有"]
						a        =     "".join(lowerCamelCase_	)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																a        =     True
																a        =     self.tokenizer_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)
																a        =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)

																a        =     tokenizer_p.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
																a        =     tokenizer_r.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)

																a        =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_	)
																a        =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_	)

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)
																self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)

																a        =     False
																a        =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)
																a        =     self.tokenizer_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)

																a        =     tokenizer_r.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)
																a        =     tokenizer_p.encode(lowerCamelCase_	, add_special_tokens=lowerCamelCase_	)

																a        =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_	)
																a        =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_	)

																# it is expected that only the first Chinese character is not preceded by "##".
																a        =     [
																    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase_	)
																]
																self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)
																self.assertListEqual(lowerCamelCase_	, lowerCamelCase_	)





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
				import torch

				from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer



@require_torch
class 			_lowercase							(     unittest.TestCase			):





	"""simple docstring"""




	@slow
	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     XLMProphetNetForConditionalGeneration.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"	)
						model.to(lowerCamelCase_	)

						# encoder-decoder outputs
						a        =     torch.tensor([[17, 96208, 103471, 2]]	).to(lowerCamelCase_	)
						a        =     torch.tensor(
						    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]	).to(lowerCamelCase_	)
						a        =     model(
						    input_ids=lowerCamelCase_	, attention_mask=lowerCamelCase_	, encoder_outputs=lowerCamelCase_	, decoder_input_ids=lowerCamelCase_	)
						a        =     output[0]
						a        =     torch.Size((1, 14, 250012)	)
						self.assertEqual(output_predited_logis.shape	, lowerCamelCase_	)
						a        =     torch.tensor(
						    [[[-6.3986, -8.2391, 12.5189], [-6.3289, -8.0864, 12.6211], [-6.2418, -8.0445, 12.7968]]]	).to(lowerCamelCase_	)
						self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]	, lowerCamelCase_	, atol=1E-4	)	)

						# encoder outputs
						a        =     model.prophetnet.encoder(lowerCamelCase_	)[0]
						a        =     torch.tensor(
						    [[[-1.4260, -0.7628, 0.8453], [-1.4719, -0.1391, 0.7807], [-1.7678, 0.0114, 0.4646]]]	).to(lowerCamelCase_	)
						a        =     torch.Size((1, 4, 1024)	)
						self.assertEqual(encoder_outputs.shape	, lowerCamelCase_	)
						self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]	, lowerCamelCase_	, atol=1E-4	)	)

						# decoder outputs
						a        =     model.prophetnet.decoder(
						    lowerCamelCase_	, encoder_hidden_states=lowerCamelCase_	, )
						a        =     decoder_outputs[1].view(1	, model.config.ngram	, 14	, -1	)
						a        =     model.lm_head(lowerCamelCase_	)
						a        =     predicting_streams_logits[:, 0]
						self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]	, lowerCamelCase_	, atol=1E-4	)	)




	@slow
	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     XLMProphetNetForConditionalGeneration.from_pretrained(
						    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"	)
						model.to(lowerCamelCase_	)

						a        =     torch.tensor([[17, 96208, 103471, 2]]	).to(lowerCamelCase_	)
						a        =     torch.tensor(
						    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]	).to(lowerCamelCase_	)
						a        =     model(
						    input_ids=lowerCamelCase_	, attention_mask=lowerCamelCase_	, encoder_outputs=lowerCamelCase_	, decoder_input_ids=lowerCamelCase_	)
						a        =     output[0]
						a        =     torch.Size((1, 14, 250012)	)
						self.assertEqual(output_predited_logis.shape	, lowerCamelCase_	)
						# compare the actual values for a slice.
						a        =     torch.tensor(
						    [[[-9.2253, -9.7173, -6.3529], [-7.6701, -9.0145, -1.9382], [-8.0195, -7.0004, -0.1523]]]	).to(lowerCamelCase_	)

						self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]	, lowerCamelCase_	, atol=1E-4	)	)




	@slow
	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     XLMProphetNetForConditionalGeneration.from_pretrained(
						    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"	)
						model.to(lowerCamelCase_	)
						a        =     512

						a        =     XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"	)

						a        =     (
						    "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"
						    " January 14, 2020, according to the official portal of the organization. From that day, users of this"
						    " system will not be able to receive security updates, which could make their computers vulnerable to"
						    " cyber attacks."
						)
						a        =     (
						    "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"
						    " после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"
						    " этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"
						    " к кибератакам."
						)
						a        =     (
						    "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
						)

						a        =     tokenizer(
						    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]	, padding=lowerCamelCase_	, max_length=255	, return_tensors="pt"	).input_ids
						a        =     input_ids.to(lowerCamelCase_	)

						a        =     model.generate(
						    lowerCamelCase_	, num_beams=10	, length_penalty=1.0	, no_repeat_ngram_size=3	, early_stopping=lowerCamelCase_	)
						a        =     [tokenizer.decode(lowerCamelCase_	, skip_special_tokens=lowerCamelCase_	) for g in summary_ids]
						a        =     "Microsoft to end Windows 7 free support after January 14, 2020"
						a        =     "Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"
						a        =     "微软打算终止对Windows 7操作系统的免费支持"
						self.assertListEqual(
						    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]	, lowerCamelCase_	, )

						a        =     model.generate(
						    lowerCamelCase_	, num_beams=1	, length_penalty=1.0	, no_repeat_ngram_size=3	, early_stopping=lowerCamelCase_	)
						a        =     [
						    tokenizer.convert_ids_to_tokens(lowerCamelCase_	, skip_special_tokens=lowerCamelCase_	) for g in summary_ids_beama
						]
						a        =     "▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7".split(" "	)
						a        =     "▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года".split(
						    " "	)
						a        =     "微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持".split(" "	)
						self.assertListEqual(
						    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]	, lowerCamelCase_	, )
