





"""simple docstring"""



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 	__snake_case      (	A_							,							unittest.TestCase  ):





 """simple docstring"""


 _lowerCamelCase				 =  ElectraTokenizer
 _lowerCamelCase				 =  ElectraTokenizerFast
 _lowerCamelCase				 =  True
 _lowerCamelCase				 =  True
 _lowerCamelCase				 =  filter_non_english

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     super().setUp()

     __A					:		List[str]						=  [
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     __A					:		str						=  os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES['''vocab_file''']						)
     with open(self.vocab_file      ,		'''w'''      ,		encoding='''utf-8'''						) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)

 def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  '''UNwant\u00E9d,running'''
     __A					:		List[Any]						=  '''unwanted, running'''
     return input_text, output_text

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[Any]						=  self.tokenizer_class(self.vocab_file						)

     __A					:		Dict						=  tokenizer.tokenize('''UNwant\u00E9d,running'''						)
     self.assertListEqual(__lowerCamelCase      ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']						)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)      ,		[9, 6, 7, 12, 10, 11]						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     if not self.test_rust_tokenizer:
         return

     __A					:		Tuple						=  self.get_tokenizer()
     __A					:		Dict						=  self.get_rust_tokenizer()

     __A					:		int						=  '''UNwant\u00E9d,running'''

     __A					:		str						=  tokenizer.tokenize(__lowerCamelCase						)
     __A					:		str						=  rust_tokenizer.tokenize(__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

     __A					:		Any						=  tokenizer.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
     __A					:		List[Any]						=  rust_tokenizer.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

     __A					:		Union[str, Any]						=  self.get_rust_tokenizer()
     __A					:		Optional[int]						=  tokenizer.encode(__lowerCamelCase						)
     __A					:		Union[str, Any]						=  rust_tokenizer.encode(__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

     # With lower casing
     __A					:		Any						=  self.get_tokenizer(do_lower_case=__lowerCamelCase						)
     __A					:		Tuple						=  self.get_rust_tokenizer(do_lower_case=__lowerCamelCase						)

     __A					:		List[str]						=  '''UNwant\u00E9d,running'''

     __A					:		Tuple						=  tokenizer.tokenize(__lowerCamelCase						)
     __A					:		int						=  rust_tokenizer.tokenize(__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

     __A					:		List[Any]						=  tokenizer.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
     __A					:		Tuple						=  rust_tokenizer.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

     __A					:		Union[str, Any]						=  self.get_rust_tokenizer()
     __A					:		Tuple						=  tokenizer.encode(__lowerCamelCase						)
     __A					:		Union[str, Any]						=  rust_tokenizer.encode(__lowerCamelCase						)
     self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Dict						=  BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''						)      ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  BasicTokenizer(do_lower_case=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''						)      ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)      ,		['''hello''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Any						=  BasicTokenizer(do_lower_case=__lowerCamelCase      ,		strip_accents=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)      ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)      ,		['''h\u00E9llo''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Dict						=  BasicTokenizer(do_lower_case=__lowerCamelCase      ,		strip_accents=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)      ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)      ,		['''hello''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		int						=  BasicTokenizer(do_lower_case=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)      ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)      ,		['''hello''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  BasicTokenizer(do_lower_case=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''						)      ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[Any]						=  BasicTokenizer(do_lower_case=__lowerCamelCase      ,		strip_accents=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)      ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  BasicTokenizer(do_lower_case=__lowerCamelCase      ,		strip_accents=__lowerCamelCase						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)      ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Tuple						=  BasicTokenizer(do_lower_case=__lowerCamelCase      ,		never_split=['''[UNK]''']						)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''						)      ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		str						=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

     __A					:		List[str]						=  {}
     for i, token in enumerate(__lowerCamelCase						):
         __A					:		Any						=  i
     __A					:		List[Any]						=  WordpieceTokenizer(vocab=__lowerCamelCase      ,		unk_token='''[UNK]'''						)

     self.assertListEqual(tokenizer.tokenize(''''''						)      ,		[]						)

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''						)      ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']						)

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''						)      ,		['''[UNK]''', '''runn''', '''##ing''']						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     self.assertTrue(_is_whitespace(''' '''						)						)
     self.assertTrue(_is_whitespace('''\t'''						)						)
     self.assertTrue(_is_whitespace('''\r'''						)						)
     self.assertTrue(_is_whitespace('''\n'''						)						)
     self.assertTrue(_is_whitespace('''\u00A0'''						)						)

     self.assertFalse(_is_whitespace('''A'''						)						)
     self.assertFalse(_is_whitespace('''-'''						)						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     self.assertTrue(_is_control('''\u0005'''						)						)

     self.assertFalse(_is_control('''A'''						)						)
     self.assertFalse(_is_control(''' '''						)						)
     self.assertFalse(_is_control('''\t'''						)						)
     self.assertFalse(_is_control('''\r'''						)						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     self.assertTrue(_is_punctuation('''-'''						)						)
     self.assertTrue(_is_punctuation('''$'''						)						)
     self.assertTrue(_is_punctuation('''`'''						)						)
     self.assertTrue(_is_punctuation('''.'''						)						)

     self.assertFalse(_is_punctuation('''A'''						)						)
     self.assertFalse(_is_punctuation(''' '''						)						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[Any]						=  self.get_tokenizer()
     __A					:		Tuple						=  self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(__lowerCamelCase						) for t in ['''Test''', '''\xad''', '''test''']]      ,		[['''[UNK]'''], [], ['''[UNK]''']]						)

     self.assertListEqual(
         [rust_tokenizer.tokenize(__lowerCamelCase						) for t in ['''Test''', '''\xad''', '''test''']]      ,		[['''[UNK]'''], [], ['''[UNK]''']]						)

 @slow
 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''						)

     __A					:		Dict						=  tokenizer.encode('''sequence builders'''      ,		add_special_tokens=__lowerCamelCase						)
     __A					:		Tuple						=  tokenizer.encode('''multi-sequence build'''      ,		add_special_tokens=__lowerCamelCase						)

     __A					:		Tuple						=  tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
     __A					:		str						=  tokenizer.build_inputs_with_special_tokens(__lowerCamelCase      ,		__lowerCamelCase						)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
             __A					:		Dict						=  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase      ,		**__lowerCamelCase						)

             __A					:		Tuple						=  F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             __A					:		Optional[Any]						=  tokenizer_r.encode_plus(
                 __lowerCamelCase      ,		return_attention_mask=__lowerCamelCase      ,		return_token_type_ids=__lowerCamelCase      ,		return_offsets_mapping=__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase      ,		)

             __A					:		List[Any]						=  tokenizer_r.do_lower_case if hasattr(__lowerCamelCase      ,		'''do_lower_case'''						) else False
             __A					:		Union[str, Any]						=  (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), '''A'''),
                     ((1, 2), ''','''),
                     ((3, 5), '''na'''),
                     ((5, 6), '''##ï'''),
                     ((6, 8), '''##ve'''),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), '''Allen'''),
                     ((21, 23), '''##NL'''),
                     ((23, 24), '''##P'''),
                     ((25, 33), '''sentence'''),
                     ((33, 34), '''.'''),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), '''a'''),
                     ((1, 2), ''','''),
                     ((3, 8), '''naive'''),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), '''allen'''),
                     ((21, 23), '''##nl'''),
                     ((23, 24), '''##p'''),
                     ((25, 33), '''sentence'''),
                     ((33, 34), '''.'''),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]      ,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']						)						)
             self.assertEqual([e[0] for e in expected_results]      ,		tokens['''offset_mapping''']						)







 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Union[str, Any]						=  ['''的''', '''人''', '''有''']
     __A					:		Dict						=  ''''''.join(__lowerCamelCase						)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
             __A					:		Optional[int]						=  True
             __A					:		str						=  self.tokenizer_class.from_pretrained(__lowerCamelCase      ,		**__lowerCamelCase						)
             __A					:		Tuple						=  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase      ,		**__lowerCamelCase						)

             __A					:		List[Any]						=  tokenizer_p.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
             __A					:		List[str]						=  tokenizer_r.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)

             __A					:		Optional[int]						=  tokenizer_r.convert_ids_to_tokens(__lowerCamelCase						)
             __A					:		Union[str, Any]						=  tokenizer_p.convert_ids_to_tokens(__lowerCamelCase						)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)
             self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)

             __A					:		List[Any]						=  False
             __A					:		List[str]						=  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase      ,		**__lowerCamelCase						)
             __A					:		Optional[int]						=  self.tokenizer_class.from_pretrained(__lowerCamelCase      ,		**__lowerCamelCase						)

             __A					:		Any						=  tokenizer_r.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)
             __A					:		Optional[int]						=  tokenizer_p.encode(__lowerCamelCase      ,		add_special_tokens=__lowerCamelCase						)

             __A					:		str						=  tokenizer_r.convert_ids_to_tokens(__lowerCamelCase						)
             __A					:		List[Any]						=  tokenizer_p.convert_ids_to_tokens(__lowerCamelCase						)

             # it is expected that only the first Chinese character is not preceded by "##".
             __A					:		Dict						=  [
                 F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase						)
             ]
             self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)
             self.assertListEqual(__lowerCamelCase      ,		__lowerCamelCase						)











"""simple docstring"""



import hashlib
import importlib.util
import json
import os
import pathlib
from types import ModuleType

import pytest
import requests

a_				=    pathlib.Path.cwd().joinpath("""project_euler""")
a_				=    pathlib.Path.cwd().joinpath(
    """scripts""", """project_euler_answers.json"""
)

with open(PROJECT_EULER_ANSWERS_PATH) as file_handle:
       a_				=    json.load(file_handle)







def 	__lowercase  (     snake_case_						:						pathlib.Path		)     ->ModuleType:



    '''simple docstring'''







    __A					:		str						=  importlib.util.spec_from_file_location(file_path.name       ,str(snake_case_		)		)
    __A					:		Optional[int]						=  importlib.util.module_from_spec(snake_case_		)  # type: ignore
    spec.loader.exec_module(snake_case_		)  # type: ignore
    return module







def 	__lowercase  (     )     ->list[pathlib.Path]:



    '''simple docstring'''







    __A					:		Optional[int]						=  []
    for problem_dir_path in PROJECT_EULER_DIR_PATH.iterdir():
        if problem_dir_path.is_file() or problem_dir_path.name.startswith('''_'''		):
            continue
        for file_path in problem_dir_path.iterdir():
            if file_path.suffix != ".py" or file_path.name.startswith(('''_''', '''test''')		):
                continue
            solution_file_paths.append(snake_case_		)
    return solution_file_paths







def 	__lowercase  (     )     ->str:



    '''simple docstring'''







    with open(os.environ['''GITHUB_EVENT_PATH''']		) as file:
        __A					:		Any						=  json.load(snake_case_		)
    return event["pull_request"]["url"] + "/files"







def 	__lowercase  (     )     ->list[pathlib.Path]:



    '''simple docstring'''







    __A					:		int						=  []
    __A					:		Optional[int]						=  {
        '''Accept''': '''application/vnd.github.v3+json''',
        '''Authorization''': '''token ''' + os.environ['''GITHUB_TOKEN'''],
    }
    __A					:		Optional[Any]						=  requests.get(get_files_url()       ,headers=snake_case_		).json()
    for file in files:
        __A					:		Union[str, Any]						=  pathlib.Path.cwd().joinpath(file['''filename''']		)
        if (
            filepath.suffix != ".py"
            or filepath.name.startswith(('''_''', '''test''')		)
            or not filepath.name.startswith('''sol'''		)
        ):
            continue
        solution_file_paths.append(snake_case_		)
    return solution_file_paths







def 	__lowercase  (     )     ->list[pathlib.Path]:



    '''simple docstring'''







    if os.environ.get('''CI'''		) and os.environ.get('''GITHUB_EVENT_NAME'''		) == "pull_request":
        # Return only if there are any, otherwise default to all solutions
        if filepaths := added_solution_file_path():
            return filepaths
    return all_solution_file_paths()







@pytest.mark.parametrize(
    '''solution_path'''       ,collect_solution_file_paths()       ,ids=lambda snake_case_		: F"""{path.parent.name}/{path.name}"""       ,)
def 	__lowercase  (     snake_case_						:						pathlib.Path		)     ->None:



    '''simple docstring'''







    __A					:		str						=  solution_path.parent.name[8:].zfill(3		)
    __A					:		str						=  PROBLEM_ANSWERS[problem_number]
    __A					:		Union[str, Any]						=  convert_path_to_module(snake_case_		)
    __A					:		List[str]						=  str(solution_module.solution()		)  # type: ignore
    __A					:		List[str]						=  hashlib.shaaaa(answer.encode()		).hexdigest()
    assert (
        answer == expected
    ), F"""Expected solution to {problem_number} to have hash {expected}, got {answer}"""




