
"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 					A_			(  A__   ,      unittest.TestCase     ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_			   =     ElectraTokenizer
							SCREAMING_SNAKE_CASE_			   =     ElectraTokenizerFast
							SCREAMING_SNAKE_CASE_			   =     True
							SCREAMING_SNAKE_CASE_			   =     True
							SCREAMING_SNAKE_CASE_			   =     filter_non_english





							def    UpperCAmelCase__							(						self						:str ):





										"""simple docstring"""

										super().setUp()

										lowerCamelCase__			:  int  =[
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										lowerCamelCase__			:  int  =os.path.join(self.tmpdirname  ,							VOCAB_FILES_NAMES['vocab_file'] )
										with open(self.vocab_file  ,							'w'  ,							encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )





							def    UpperCAmelCase__							(						self						:Optional[Any]  ,							lowerCamelCase_						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  int  ="UNwant\u00E9d,running"
										lowerCamelCase__			:  Dict  ="unwanted, running"
										return input_text, output_text





							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  Union[str, Any]  =self.tokenizer_class(self.vocab_file )

										lowerCamelCase__			:  int  =tokenizer.tokenize('UNwant\u00E9d,running' )
										self.assertListEqual(lowerCamelCase_  ,							['un', '##want', '##ed', ',', 'runn', '##ing'] )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_ )  ,							[9, 6, 7, 12, 10, 11] )





							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										if not self.test_rust_tokenizer:
													return

										lowerCamelCase__			:  Any  =self.get_tokenizer()
										lowerCamelCase__			:  Dict  =self.get_rust_tokenizer()

										lowerCamelCase__			:  str  ="UNwant\u00E9d,running"

										lowerCamelCase__			:  Tuple  =tokenizer.tokenize(lowerCamelCase_ )
										lowerCamelCase__			:  Optional[Any]  =rust_tokenizer.tokenize(lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

										lowerCamelCase__			:  str  =tokenizer.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
										lowerCamelCase__			:  str  =rust_tokenizer.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

										lowerCamelCase__			:  Optional[Any]  =self.get_rust_tokenizer()
										lowerCamelCase__			:  List[Any]  =tokenizer.encode(lowerCamelCase_ )
										lowerCamelCase__			:  Optional[int]  =rust_tokenizer.encode(lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

										# With lower casing
										lowerCamelCase__			:  Union[str, Any]  =self.get_tokenizer(do_lower_case=lowerCamelCase_ )
										lowerCamelCase__			:  Dict  =self.get_rust_tokenizer(do_lower_case=lowerCamelCase_ )

										lowerCamelCase__			:  Any  ="UNwant\u00E9d,running"

										lowerCamelCase__			:  Tuple  =tokenizer.tokenize(lowerCamelCase_ )
										lowerCamelCase__			:  Optional[Any]  =rust_tokenizer.tokenize(lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

										lowerCamelCase__			:  str  =tokenizer.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
										lowerCamelCase__			:  Optional[Any]  =rust_tokenizer.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

										lowerCamelCase__			:  List[Any]  =self.get_rust_tokenizer()
										lowerCamelCase__			:  List[str]  =tokenizer.encode(lowerCamelCase_ )
										lowerCamelCase__			:  Any  =rust_tokenizer.encode(lowerCamelCase_ )
										self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )





							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  Any  =BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )  ,							['ah', '\u535A', '\u63A8', 'zz'] )





							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  str  =BasicTokenizer(do_lower_case=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )  ,							['hello', '!', 'how', 'are', 'you', '?'] )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )  ,							['hello'] )





							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										lowerCamelCase__			:  int  =BasicTokenizer(do_lower_case=lowerCamelCase_  ,							strip_accents=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )  ,							['hällo', '!', 'how', 'are', 'you', '?'] )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )  ,							['h\u00E9llo'] )





							def    UpperCAmelCase__							(						self						:Dict ):





										"""simple docstring"""

										lowerCamelCase__			:  Dict  =BasicTokenizer(do_lower_case=lowerCamelCase_  ,							strip_accents=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )  ,							['hallo', '!', 'how', 'are', 'you', '?'] )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )  ,							['hello'] )





							def    UpperCAmelCase__							(						self						:str ):





										"""simple docstring"""

										lowerCamelCase__			:  List[Any]  =BasicTokenizer(do_lower_case=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )  ,							['hallo', '!', 'how', 'are', 'you', '?'] )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )  ,							['hello'] )





							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  Union[str, Any]  =BasicTokenizer(do_lower_case=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )  ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )





							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  Optional[Any]  =BasicTokenizer(do_lower_case=lowerCamelCase_  ,							strip_accents=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )  ,							['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )





							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										lowerCamelCase__			:  str  =BasicTokenizer(do_lower_case=lowerCamelCase_  ,							strip_accents=lowerCamelCase_ )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )  ,							['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )





							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  int  =BasicTokenizer(do_lower_case=lowerCamelCase_  ,							never_split=['[UNK]'] )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )  ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )





							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										lowerCamelCase__			:  Union[str, Any]  =["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										lowerCamelCase__			:  Tuple  ={}
										for i, token in enumerate(lowerCamelCase_ ):
													lowerCamelCase__			:  str  =i
										lowerCamelCase__			:  Any  =WordpieceTokenizer(vocab=lowerCamelCase_  ,							unk_token='[UNK]' )

										self.assertListEqual(tokenizer.tokenize('' )  ,							[] )

										self.assertListEqual(tokenizer.tokenize('unwanted running' )  ,							['un', '##want', '##ed', 'runn', '##ing'] )

										self.assertListEqual(tokenizer.tokenize('unwantedX running' )  ,							['[UNK]', 'runn', '##ing'] )





							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										self.assertTrue(_is_whitespace(' ' ) )
										self.assertTrue(_is_whitespace('\t' ) )
										self.assertTrue(_is_whitespace('\r' ) )
										self.assertTrue(_is_whitespace('\n' ) )
										self.assertTrue(_is_whitespace('\u00A0' ) )

										self.assertFalse(_is_whitespace('A' ) )
										self.assertFalse(_is_whitespace('-' ) )





							def    UpperCAmelCase__							(						self						:Tuple ):





										"""simple docstring"""

										self.assertTrue(_is_control('\u0005' ) )

										self.assertFalse(_is_control('A' ) )
										self.assertFalse(_is_control(' ' ) )
										self.assertFalse(_is_control('\t' ) )
										self.assertFalse(_is_control('\r' ) )





							def    UpperCAmelCase__							(						self						:Union[str, Any] ):





										"""simple docstring"""

										self.assertTrue(_is_punctuation('-' ) )
										self.assertTrue(_is_punctuation('$' ) )
										self.assertTrue(_is_punctuation('`' ) )
										self.assertTrue(_is_punctuation('.' ) )

										self.assertFalse(_is_punctuation('A' ) )
										self.assertFalse(_is_punctuation(' ' ) )





							def    UpperCAmelCase__							(						self						:Any ):





										"""simple docstring"""

										lowerCamelCase__			:  Tuple  =self.get_tokenizer()
										lowerCamelCase__			:  Dict  =self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCamelCase_ ) for t in ['Test', '\xad', 'test']]  ,							[['[UNK]'], [], ['[UNK]']] )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCamelCase_ ) for t in ['Test', '\xad', 'test']]  ,							[['[UNK]'], [], ['[UNK]']] )





							@slow
							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  List[str]  =self.tokenizer_class.from_pretrained('google/electra-base-discriminator' )

										lowerCamelCase__			:  Dict  =tokenizer.encode('sequence builders'  ,							add_special_tokens=lowerCamelCase_ )
										lowerCamelCase__			:  Optional[Any]  =tokenizer.encode('multi-sequence build'  ,							add_special_tokens=lowerCamelCase_ )

										lowerCamelCase__			:  str  =tokenizer.build_inputs_with_special_tokens(lowerCamelCase_ )
										lowerCamelCase__			:  Tuple  =tokenizer.build_inputs_with_special_tokens(lowerCamelCase_  ,							lowerCamelCase_ )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]





							def    UpperCAmelCase__							(						self						:Optional[Any] ):





										"""simple docstring"""

										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																lowerCamelCase__			:  Any  =self.rust_tokenizer_class.from_pretrained(lowerCamelCase_  ,							**lowerCamelCase_ )

																lowerCamelCase__			:  Optional[Any]  =f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																lowerCamelCase__			:  Union[str, Any]  =tokenizer_r.encode_plus(
																    lowerCamelCase_  ,							return_attention_mask=lowerCamelCase_  ,							return_token_type_ids=lowerCamelCase_  ,							return_offsets_mapping=lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_  ,							)

																lowerCamelCase__			:  Optional[int]  =tokenizer_r.do_lower_case if hasattr(lowerCamelCase_  ,							'do_lower_case' ) else False
																lowerCamelCase__			:  Tuple  =(
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "A"),
																        ((1, 2), ","),
																        ((3, 5), "na"),
																        ((5, 6), "##ï"),
																        ((6, 8), "##ve"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "Allen"),
																        ((21, 23), "##NL"),
																        ((23, 24), "##P"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "a"),
																        ((1, 2), ","),
																        ((3, 8), "naive"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "allen"),
																        ((21, 23), "##nl"),
																        ((23, 24), "##p"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]  ,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
																self.assertEqual([e[0] for e in expected_results]  ,							tokens['offset_mapping'] )





							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  Union[str, Any]  =["的", "人", "有"]
										lowerCamelCase__			:  Optional[int]  ="".join(lowerCamelCase_ )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																lowerCamelCase__			:  List[Any]  =True
																lowerCamelCase__			:  str  =self.tokenizer_class.from_pretrained(lowerCamelCase_  ,							**lowerCamelCase_ )
																lowerCamelCase__			:  Tuple  =self.rust_tokenizer_class.from_pretrained(lowerCamelCase_  ,							**lowerCamelCase_ )

																lowerCamelCase__			:  Union[str, Any]  =tokenizer_p.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
																lowerCamelCase__			:  Optional[Any]  =tokenizer_r.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )

																lowerCamelCase__			:  Any  =tokenizer_r.convert_ids_to_tokens(lowerCamelCase_ )
																lowerCamelCase__			:  Any  =tokenizer_p.convert_ids_to_tokens(lowerCamelCase_ )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )
																self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

																lowerCamelCase__			:  Tuple  =False
																lowerCamelCase__			:  Tuple  =self.rust_tokenizer_class.from_pretrained(lowerCamelCase_  ,							**lowerCamelCase_ )
																lowerCamelCase__			:  List[str]  =self.tokenizer_class.from_pretrained(lowerCamelCase_  ,							**lowerCamelCase_ )

																lowerCamelCase__			:  Optional[Any]  =tokenizer_r.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )
																lowerCamelCase__			:  Optional[Any]  =tokenizer_p.encode(lowerCamelCase_  ,							add_special_tokens=lowerCamelCase_ )

																lowerCamelCase__			:  List[Any]  =tokenizer_r.convert_ids_to_tokens(lowerCamelCase_ )
																lowerCamelCase__			:  Union[str, Any]  =tokenizer_p.convert_ids_to_tokens(lowerCamelCase_ )

																# it is expected that only the first Chinese character is not preceded by "##".
																lowerCamelCase__			:  Optional[int]  =[
																    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase_ )
																]
																self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )
																self.assertListEqual(lowerCamelCase_  ,							lowerCamelCase_ )

"""simple docstring"""


from sklearn.neural_network import MLPClassifier

lowerCAmelCase			=		[[0.0, 0.0], [1.0, 1.0], [1.0, 0.0], [0.0, 1.0]]
lowerCAmelCase			=		[0, 1, 0, 0]


lowerCAmelCase			=		MLPClassifier(
    solver="""lbfgs""", alpha=1E-5, hidden_layer_sizes=(5, 2), random_state=1
)

clf.fit(X, y)


lowerCAmelCase			=		[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0]]
lowerCAmelCase			=		clf.predict(test)






def      lowerCAmelCase_     (	snake_case_	:    Tuple    )     ->List[Any]:
			return list(snake_case_    )


if __name__ == "__main__":
							import doctest

							doctest.testmod()