






"""simple docstring"""



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class        __lowercase(       _UpperCamelCase						,							unittest.TestCase  ):
						lowerCAmelCase_		:							Dict=       RealmTokenizer
						lowerCAmelCase_		:							List[Any]=       RealmTokenizerFast
						lowerCAmelCase_		:							Optional[Any]=       True
						lowerCAmelCase_		:							Any=       True
						lowerCAmelCase_		:							str=       filter_non_english






						def 		lowerCAmelCase   (					self)			->  Dict:




											"""simple docstring"""





											super().setUp()

											lowerCamelCase__				=[
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											lowerCamelCase__				=os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['vocab_file'])
											with open(self.vocab_file    ,						'w'    ,						encoding='utf-8') as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))






						def 		lowerCAmelCase   (					self    ,						_SCREAMING_SNAKE_CASE)			->  Optional[Any]:




											"""simple docstring"""





											lowerCamelCase__				='UNwant\u00E9d,running'
											lowerCamelCase__				='unwanted, running'
											return input_text, output_text






						def 		lowerCAmelCase   (					self)			->  Optional[int]:




											"""simple docstring"""





											lowerCamelCase__				=self.tokenizer_class(self.vocab_file)

											lowerCamelCase__				=tokenizer.tokenize('UNwant\u00E9d,running')
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						['un', '##want', '##ed', ',', 'runn', '##ing'])
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)    ,						[9, 6, 7, 12, 10, 11])






						def 		lowerCAmelCase   (					self)			->  Dict:




											"""simple docstring"""





											if not self.test_rust_tokenizer:
																return

											lowerCamelCase__				=self.get_tokenizer()
											lowerCamelCase__				=self.get_rust_tokenizer()

											lowerCamelCase__				='UNwant\u00E9d,running'

											lowerCamelCase__				=tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=self.get_rust_tokenizer()
											lowerCamelCase__				=tokenizer.encode(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.encode(_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											# With lower casing
											lowerCamelCase__				=self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				='UNwant\u00E9d,running'

											lowerCamelCase__				=tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=self.get_rust_tokenizer()
											lowerCamelCase__				=tokenizer.encode(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=rust_tokenizer.encode(_SCREAMING_SNAKE_CASE)
											self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)






						def 		lowerCAmelCase   (					self)			->  Any:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')    ,						['ah', '\u535A', '\u63A8', 'zz'])






						def 		lowerCAmelCase   (					self)			->  List[Any]:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,						['hello', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,						['hello'])






						def 		lowerCAmelCase   (					self)			->  Optional[Any]:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    ,						strip_accents=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,						['hällo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,						['h\u00E9llo'])






						def 		lowerCAmelCase   (					self)			->  Optional[int]:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    ,						strip_accents=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,						['hallo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,						['hello'])






						def 		lowerCAmelCase   (					self)			->  Tuple:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,						['hallo', '!', 'how', 'are', 'you', '?'])
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,						['hello'])






						def 		lowerCAmelCase   (					self)			->  List[str]:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?'])






						def 		lowerCAmelCase   (					self)			->  Tuple:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    ,						strip_accents=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?'])






						def 		lowerCAmelCase   (					self)			->  List[Any]:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    ,						strip_accents=_SCREAMING_SNAKE_CASE)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?'])






						def 		lowerCAmelCase   (					self)			->  Dict:




											"""simple docstring"""





											lowerCamelCase__				=BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE    ,						never_split=['[UNK]'])

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')    ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])






						def 		lowerCAmelCase   (					self)			->  List[str]:




											"""simple docstring"""





											lowerCamelCase__				=['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											lowerCamelCase__				={}
											for i, token in enumerate(_SCREAMING_SNAKE_CASE):
																lowerCamelCase__				=i
											lowerCamelCase__				=WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE    ,						unk_token='[UNK]')

											self.assertListEqual(tokenizer.tokenize('')    ,						[])

											self.assertListEqual(tokenizer.tokenize('unwanted running')    ,						['un', '##want', '##ed', 'runn', '##ing'])

											self.assertListEqual(tokenizer.tokenize('unwantedX running')    ,						['[UNK]', 'runn', '##ing'])






						def 		lowerCAmelCase   (					self)			->  Any:




											"""simple docstring"""





											self.assertTrue(_is_whitespace(' '))
											self.assertTrue(_is_whitespace('\t'))
											self.assertTrue(_is_whitespace('\r'))
											self.assertTrue(_is_whitespace('\n'))
											self.assertTrue(_is_whitespace('\u00A0'))

											self.assertFalse(_is_whitespace('A'))
											self.assertFalse(_is_whitespace('-'))






						def 		lowerCAmelCase   (					self)			->  Optional[int]:




											"""simple docstring"""





											self.assertTrue(_is_control('\u0005'))

											self.assertFalse(_is_control('A'))
											self.assertFalse(_is_control(' '))
											self.assertFalse(_is_control('\t'))
											self.assertFalse(_is_control('\r'))






						def 		lowerCAmelCase   (					self)			->  List[str]:




											"""simple docstring"""





											self.assertTrue(_is_punctuation('-'))
											self.assertTrue(_is_punctuation('$'))
											self.assertTrue(_is_punctuation('`'))
											self.assertTrue(_is_punctuation('.'))

											self.assertFalse(_is_punctuation('A'))
											self.assertFalse(_is_punctuation(' '))






						def 		lowerCAmelCase   (					self)			->  int:




											"""simple docstring"""





											lowerCamelCase__				=self.get_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ['Test', '\xad', 'test']]    ,						[['[UNK]'], [], ['[UNK]']])

											if self.test_rust_tokenizer:
																lowerCamelCase__				=self.get_rust_tokenizer()
																self.assertListEqual(
																    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ['Test', '\xad', 'test']]    ,						[['[UNK]'], [], ['[UNK]']])






						@slow
						def 		lowerCAmelCase   (					self)			->  Union[str, Any]:




											"""simple docstring"""





											lowerCamelCase__				=self.tokenizer_class.from_pretrained('bert-base-uncased')

											lowerCamelCase__				=tokenizer.encode('sequence builders'    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.encode('multi-sequence build'    ,						add_special_tokens=_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE)

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






						def 		lowerCAmelCase   (					self)			->  List[Any]:




											"""simple docstring"""





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
																					lowerCamelCase__				=self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)

																					lowerCamelCase__				=F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																					lowerCamelCase__				=tokenizer_r.encode_plus(
																					    _SCREAMING_SNAKE_CASE    ,						return_attention_mask=_SCREAMING_SNAKE_CASE    ,						return_token_type_ids=_SCREAMING_SNAKE_CASE    ,						return_offsets_mapping=_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE    ,						)

																					lowerCamelCase__				=tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE    ,						'do_lower_case') else False
																					lowerCamelCase__				=(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), 'A'),
																					        ((1, 2), ','),
																					        ((3, 5), 'na'),
																					        ((5, 6), '##ï'),
																					        ((6, 8), '##ve'),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), 'Allen'),
																					        ((21, 23), '##NL'),
																					        ((23, 24), '##P'),
																					        ((25, 33), 'sentence'),
																					        ((33, 34), '.'),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), 'a'),
																					        ((1, 2), ','),
																					        ((3, 8), 'naive'),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), 'allen'),
																					        ((21, 23), '##nl'),
																					        ((23, 24), '##p'),
																					        ((25, 33), 'sentence'),
																					        ((33, 34), '.'),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]    ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
																					self.assertEqual([e[0] for e in expected_results]    ,						tokens['offset_mapping'])






						@slow
						def 		lowerCAmelCase   (					self)			->  int:




											"""simple docstring"""





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
																					lowerCamelCase__				=self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)
																					lowerCamelCase__				=self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)
																					lowerCamelCase__				=[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																					lowerCamelCase__				=tokenizer_r.batch_encode_candidates(_SCREAMING_SNAKE_CASE    ,						max_length=10    ,						return_tensors='np')
																					lowerCamelCase__				=tokenizer_p.batch_encode_candidates(_SCREAMING_SNAKE_CASE    ,						max_length=10    ,						return_tensors='np')

																					lowerCamelCase__				=(2, 2, 10)

																					self.assertEqual(encoded_sentence_r['input_ids'].shape    ,						_SCREAMING_SNAKE_CASE)
																					self.assertEqual(encoded_sentence_r['attention_mask'].shape    ,						_SCREAMING_SNAKE_CASE)
																					self.assertEqual(encoded_sentence_r['token_type_ids'].shape    ,						_SCREAMING_SNAKE_CASE)

																					self.assertEqual(encoded_sentence_p['input_ids'].shape    ,						_SCREAMING_SNAKE_CASE)
																					self.assertEqual(encoded_sentence_p['attention_mask'].shape    ,						_SCREAMING_SNAKE_CASE)
																					self.assertEqual(encoded_sentence_p['token_type_ids'].shape    ,						_SCREAMING_SNAKE_CASE)







"""simple docstring"""



_snake_case       :      Dict       =					[0.485, 0.456, 0.406]
_snake_case       :      int       =					[0.229, 0.224, 0.225]
_snake_case       :      Optional[int]       =					[0.5, 0.5, 0.5]
_snake_case       :      str       =					[0.5, 0.5, 0.5]
_snake_case       :      str       =					[0.4814_5466, 0.457_8275, 0.4082_1073]
_snake_case       :      str       =					[0.2686_2954, 0.2613_0258, 0.2757_7711]