


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class    lowerCamelCase__     ( UpperCamelCase__     ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__       =    ElectraTokenizer
     SCREAMING_SNAKE_CASE__       =    ElectraTokenizerFast
     SCREAMING_SNAKE_CASE__       =    True
     SCREAMING_SNAKE_CASE__       =    True
     SCREAMING_SNAKE_CASE__       =    filter_non_english





     def   __A      (self )  ->     Tuple:
      super().setUp()

      _lowercase							   =[
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      _lowercase							   =os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES['''vocab_file'''] )
      with open(self.vocab_file	,  '''w'''	,  encoding='''utf-8''' ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )





     def   __A      (self	,  UpperCAmelCase )  ->     Tuple:
      _lowercase							   ='''UNwant\u00E9d,running'''
      _lowercase							   ='''unwanted, running'''
      return input_text, output_text





     def   __A      (self )  ->     Tuple:
      _lowercase							   =self.tokenizer_class(self.vocab_file )

      _lowercase							   =tokenizer.tokenize('''UNwant\u00E9d,running''' )
      self.assertListEqual(snake_case_	,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_ )	,  [9, 6, 7, 1_2, 1_0, 1_1] )





     def   __A      (self )  ->     Tuple:
      if not self.test_rust_tokenizer:
       return

      _lowercase							   =self.get_tokenizer()
      _lowercase							   =self.get_rust_tokenizer()

      _lowercase							   ='''UNwant\u00E9d,running'''

      _lowercase							   =tokenizer.tokenize(snake_case_ )
      _lowercase							   =rust_tokenizer.tokenize(snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )

      _lowercase							   =tokenizer.encode(snake_case_	,  add_special_tokens=snake_case_ )
      _lowercase							   =rust_tokenizer.encode(snake_case_	,  add_special_tokens=snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )

      _lowercase							   =self.get_rust_tokenizer()
      _lowercase							   =tokenizer.encode(snake_case_ )
      _lowercase							   =rust_tokenizer.encode(snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )

      # With lower casing
      _lowercase							   =self.get_tokenizer(do_lower_case=snake_case_ )
      _lowercase							   =self.get_rust_tokenizer(do_lower_case=snake_case_ )

      _lowercase							   ='''UNwant\u00E9d,running'''

      _lowercase							   =tokenizer.tokenize(snake_case_ )
      _lowercase							   =rust_tokenizer.tokenize(snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )

      _lowercase							   =tokenizer.encode(snake_case_	,  add_special_tokens=snake_case_ )
      _lowercase							   =rust_tokenizer.encode(snake_case_	,  add_special_tokens=snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )

      _lowercase							   =self.get_rust_tokenizer()
      _lowercase							   =tokenizer.encode(snake_case_ )
      _lowercase							   =rust_tokenizer.encode(snake_case_ )
      self.assertListEqual(snake_case_	,  snake_case_ )





     def   __A      (self )  ->     Dict:
      _lowercase							   =BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )	,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )





     def   __A      (self )  ->     Tuple:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )	,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )	,  ['''hello'''] )





     def   __A      (self )  ->     List[str]:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_	,  strip_accents=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )	,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )	,  ['''h\u00E9llo'''] )





     def   __A      (self )  ->     List[Any]:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_	,  strip_accents=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )	,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )	,  ['''hello'''] )





     def   __A      (self )  ->     Any:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )	,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )	,  ['''hello'''] )





     def   __A      (self )  ->     Tuple:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )	,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )





     def   __A      (self )  ->     Union[str, Any]:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_	,  strip_accents=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )	,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )





     def   __A      (self )  ->     Any:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_	,  strip_accents=snake_case_ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )	,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )





     def   __A      (self )  ->     List[str]:
      _lowercase							   =BasicTokenizer(do_lower_case=snake_case_	,  never_split=['''[UNK]'''] )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )	,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )





     def   __A      (self )  ->     str:
      _lowercase							   =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      _lowercase							   ={}
      for i, token in enumerate(snake_case_ ):
       _lowercase							   =i
      _lowercase							   =WordpieceTokenizer(vocab=snake_case_	,  unk_token='''[UNK]''' )

      self.assertListEqual(tokenizer.tokenize('''''' )	,  [] )

      self.assertListEqual(tokenizer.tokenize('''unwanted running''' )	,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

      self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )	,  ['''[UNK]''', '''runn''', '''##ing'''] )





     def   __A      (self )  ->     str:
      self.assertTrue(_is_whitespace(''' ''' ) )
      self.assertTrue(_is_whitespace('''\t''' ) )
      self.assertTrue(_is_whitespace('''\r''' ) )
      self.assertTrue(_is_whitespace('''\n''' ) )
      self.assertTrue(_is_whitespace('''\u00A0''' ) )

      self.assertFalse(_is_whitespace('''A''' ) )
      self.assertFalse(_is_whitespace('''-''' ) )





     def   __A      (self )  ->     List[Any]:
      self.assertTrue(_is_control('''\u0005''' ) )

      self.assertFalse(_is_control('''A''' ) )
      self.assertFalse(_is_control(''' ''' ) )
      self.assertFalse(_is_control('''\t''' ) )
      self.assertFalse(_is_control('''\r''' ) )





     def   __A      (self )  ->     Dict:
      self.assertTrue(_is_punctuation('''-''' ) )
      self.assertTrue(_is_punctuation('''$''' ) )
      self.assertTrue(_is_punctuation('''`''' ) )
      self.assertTrue(_is_punctuation('''.''' ) )

      self.assertFalse(_is_punctuation('''A''' ) )
      self.assertFalse(_is_punctuation(''' ''' ) )





     def   __A      (self )  ->     str:
      _lowercase							   =self.get_tokenizer()
      _lowercase							   =self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(snake_case_ ) for t in ['''Test''', '''\xad''', '''test''']]	,  [['''[UNK]'''], [], ['''[UNK]''']] )

      self.assertListEqual(
          [rust_tokenizer.tokenize(snake_case_ ) for t in ['''Test''', '''\xad''', '''test''']]	,  [['''[UNK]'''], [], ['''[UNK]''']] )





     @slow
     def   __A      (self )  ->     Union[str, Any]:
      _lowercase							   =self.tokenizer_class.from_pretrained('''google/electra-base-discriminator''' )

      _lowercase							   =tokenizer.encode('''sequence builders'''	,  add_special_tokens=snake_case_ )
      _lowercase							   =tokenizer.encode('''multi-sequence build'''	,  add_special_tokens=snake_case_ )

      _lowercase							   =tokenizer.build_inputs_with_special_tokens(snake_case_ )
      _lowercase							   =tokenizer.build_inputs_with_special_tokens(snake_case_	,  snake_case_ )

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





     def   __A      (self )  ->     Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
        _lowercase							   =self.rust_tokenizer_class.from_pretrained(snake_case_	,  **snake_case_ )

        _lowercase							   =f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
        _lowercase							   =tokenizer_r.encode_plus(
            snake_case_	,  return_attention_mask=snake_case_	,  return_token_type_ids=snake_case_	,  return_offsets_mapping=snake_case_	,  add_special_tokens=snake_case_	,  )

        _lowercase							   =tokenizer_r.do_lower_case if hasattr(snake_case_	,  '''do_lower_case''' ) else False
        _lowercase							   =(
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''A'''),
                ((1, 2), ''','''),
                ((3, 5), '''na'''),
                ((5, 6), '''##ï'''),
                ((6, 8), '''##ve'''),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), '''Allen'''),
                ((2_1, 2_3), '''##NL'''),
                ((2_3, 2_4), '''##P'''),
                ((2_5, 3_3), '''sentence'''),
                ((3_3, 3_4), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''a'''),
                ((1, 2), ''','''),
                ((3, 8), '''naive'''),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), '''allen'''),
                ((2_1, 2_3), '''##nl'''),
                ((2_3, 2_4), '''##p'''),
                ((2_5, 3_3), '''sentence'''),
                ((3_3, 3_4), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )

        self.assertEqual(
            [e[1] for e in expected_results]	,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
        self.assertEqual([e[0] for e in expected_results]	,  tokens['''offset_mapping'''] )





     def   __A      (self )  ->     Optional[Any]:
      _lowercase							   =['''的''', '''人''', '''有''']
      _lowercase							   =''''''.join(snake_case_ )
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
        _lowercase							   =True
        _lowercase							   =self.tokenizer_class.from_pretrained(snake_case_	,  **snake_case_ )
        _lowercase							   =self.rust_tokenizer_class.from_pretrained(snake_case_	,  **snake_case_ )

        _lowercase							   =tokenizer_p.encode(snake_case_	,  add_special_tokens=snake_case_ )
        _lowercase							   =tokenizer_r.encode(snake_case_	,  add_special_tokens=snake_case_ )

        _lowercase							   =tokenizer_r.convert_ids_to_tokens(snake_case_ )
        _lowercase							   =tokenizer_p.convert_ids_to_tokens(snake_case_ )

        # it is expected that each Chinese character is not preceded by "##"
        self.assertListEqual(snake_case_	,  snake_case_ )
        self.assertListEqual(snake_case_	,  snake_case_ )

        _lowercase							   =False
        _lowercase							   =self.rust_tokenizer_class.from_pretrained(snake_case_	,  **snake_case_ )
        _lowercase							   =self.tokenizer_class.from_pretrained(snake_case_	,  **snake_case_ )

        _lowercase							   =tokenizer_r.encode(snake_case_	,  add_special_tokens=snake_case_ )
        _lowercase							   =tokenizer_p.encode(snake_case_	,  add_special_tokens=snake_case_ )

        _lowercase							   =tokenizer_r.convert_ids_to_tokens(snake_case_ )
        _lowercase							   =tokenizer_p.convert_ids_to_tokens(snake_case_ )

        # it is expected that only the first Chinese character is not preceded by "##".
        _lowercase							   =[
            f"##{token}" if idx != 0 else token for idx, token in enumerate(snake_case_ )
        ]
        self.assertListEqual(snake_case_	,  snake_case_ )
        self.assertListEqual(snake_case_	,  snake_case_ )





from ..utils import DummyObject, requires_backends







class    lowerCamelCase__     ( metaclass=lowerCAmelCase):
     SCREAMING_SNAKE_CASE__       =    ['''tensorflow_text''']





     def __init__(self	,  *UpperCAmelCase	,  **UpperCAmelCase )  ->     Any:
      requires_backends(self	,  ['''tensorflow_text'''] )

