




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 		__UpperCAmelCase(		lowercase__       ,	unittest.TestCase ):





     """simple docstring"""



     __UpperCAmelCase	: Optional[int]						=    RealmTokenizer
     __UpperCAmelCase	: Any						=    RealmTokenizerFast
     __UpperCAmelCase	: Dict						=    True
     __UpperCAmelCase	: int						=    True
     __UpperCAmelCase	: Dict						=    filter_non_english

     def 				__SCREAMING_SNAKE_CASE		(      self :       int    )			->			Optional[int]:






         """simple docstring"""
         super().setUp()

         a__      =       [
             """[UNK]""",
             """[CLS]""",
             """[SEP]""",
             """[PAD]""",
             """[MASK]""",
             """want""",
             """##want""",
             """##ed""",
             """wa""",
             """un""",
             """runn""",
             """##ing""",
             """,""",
             """low""",
             """lowest""",
         ]
         a__      =       os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES["""vocab_file"""]    )
         with open(self.vocab_file						,				"""w"""						,				encoding="""utf-8"""    ) as vocab_writer:
             vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Union[str, Any]						,				A :       Any    )			->			List[str]:






         """simple docstring"""
         a__      =       """UNwant\u00E9d,running"""
         a__      =       """unwanted, running"""
         return input_text, output_text

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[int]    )			->			int:






         """simple docstring"""
         a__      =       self.tokenizer_class(self.vocab_file    )

         a__      =       tokenizer.tokenize("""UNwant\u00E9d,running"""    )
         self.assertListEqual(A						,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A    )						,				[9, 6, 7, 12, 10, 11]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[Any]    )			->			Dict:






         """simple docstring"""
         if not self.test_rust_tokenizer:
             return

         a__      =       self.get_tokenizer()
         a__      =       self.get_rust_tokenizer()

         a__      =       """UNwant\u00E9d,running"""

         a__      =       tokenizer.tokenize(A    )
         a__      =       rust_tokenizer.tokenize(A    )
         self.assertListEqual(A						,				A    )

         a__      =       tokenizer.encode(A						,				add_special_tokens=A    )
         a__      =       rust_tokenizer.encode(A						,				add_special_tokens=A    )
         self.assertListEqual(A						,				A    )

         a__      =       self.get_rust_tokenizer()
         a__      =       tokenizer.encode(A    )
         a__      =       rust_tokenizer.encode(A    )
         self.assertListEqual(A						,				A    )

         # With lower casing
         a__      =       self.get_tokenizer(do_lower_case=A    )
         a__      =       self.get_rust_tokenizer(do_lower_case=A    )

         a__      =       """UNwant\u00E9d,running"""

         a__      =       tokenizer.tokenize(A    )
         a__      =       rust_tokenizer.tokenize(A    )
         self.assertListEqual(A						,				A    )

         a__      =       tokenizer.encode(A						,				add_special_tokens=A    )
         a__      =       rust_tokenizer.encode(A						,				add_special_tokens=A    )
         self.assertListEqual(A						,				A    )

         a__      =       self.get_rust_tokenizer()
         a__      =       tokenizer.encode(A    )
         a__      =       rust_tokenizer.encode(A    )
         self.assertListEqual(A						,				A    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			Optional[int]:






         """simple docstring"""
         a__      =       BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )						,				["""ah""", """\u535A""", """\u63A8""", """zz"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Union[str, Any]    )			->			List[str]:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )						,				["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,				["""hello"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]    )			->			Tuple:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A						,				strip_accents=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,				["""h\u00E9llo"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			str:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A						,				strip_accents=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,				["""hello"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[Any]    )			->			Dict:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,				["""hello"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict    )			->			List[str]:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )						,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[Any]    )			->			Dict:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A						,				strip_accents=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       int    )			->			Optional[int]:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A						,				strip_accents=A    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[Any]    )			->			Tuple:






         """simple docstring"""
         a__      =       BasicTokenizer(do_lower_case=A						,				never_split=["""[UNK]"""]    )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )						,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       int    )			->			Optional[int]:






         """simple docstring"""
         a__      =       ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

         a__      =       {}
         for i, token in enumerate(A    ):
             a__      =       i
         a__      =       WordpieceTokenizer(vocab=A						,				unk_token="""[UNK]"""    )

         self.assertListEqual(tokenizer.tokenize(""""""    )						,				[]    )

         self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )						,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

         self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )						,				["""[UNK]""", """runn""", """##ing"""]    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Union[str, Any]    )			->			Optional[Any]:






         """simple docstring"""
         self.assertTrue(_is_whitespace(""" """    )    )
         self.assertTrue(_is_whitespace("""\t"""    )    )
         self.assertTrue(_is_whitespace("""\r"""    )    )
         self.assertTrue(_is_whitespace("""\n"""    )    )
         self.assertTrue(_is_whitespace("""\u00A0"""    )    )

         self.assertFalse(_is_whitespace("""A"""    )    )
         self.assertFalse(_is_whitespace("""-"""    )    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       Any    )			->			Tuple:






         """simple docstring"""
         self.assertTrue(_is_control("""\u0005"""    )    )

         self.assertFalse(_is_control("""A"""    )    )
         self.assertFalse(_is_control(""" """    )    )
         self.assertFalse(_is_control("""\t"""    )    )
         self.assertFalse(_is_control("""\r"""    )    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]    )			->			Dict:






         """simple docstring"""
         self.assertTrue(_is_punctuation("""-"""    )    )
         self.assertTrue(_is_punctuation("""$"""    )    )
         self.assertTrue(_is_punctuation("""`"""    )    )
         self.assertTrue(_is_punctuation("""."""    )    )

         self.assertFalse(_is_punctuation("""A"""    )    )
         self.assertFalse(_is_punctuation(""" """    )    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       int    )			->			List[str]:






         """simple docstring"""
         a__      =       self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(A    ) for t in ["""Test""", """\xad""", """test"""]]						,				[["""[UNK]"""], [], ["""[UNK]"""]]    )

         if self.test_rust_tokenizer:
             a__      =       self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(A    ) for t in ["""Test""", """\xad""", """test"""]]						,				[["""[UNK]"""], [], ["""[UNK]"""]]    )

     @slow
     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]    )			->			List[str]:






         """simple docstring"""
         a__      =       self.tokenizer_class.from_pretrained("""bert-base-uncased"""    )

         a__      =       tokenizer.encode("""sequence builders"""						,				add_special_tokens=A    )
         a__      =       tokenizer.encode("""multi-sequence build"""						,				add_special_tokens=A    )

         a__      =       tokenizer.build_inputs_with_special_tokens(A    )
         a__      =       tokenizer.build_inputs_with_special_tokens(A						,				A    )

         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[int]    )			->			Dict:






         """simple docstring"""
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
                 a__      =       self.rust_tokenizer_class.from_pretrained(A						,				**A    )

                 a__      =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                 a__      =       tokenizer_r.encode_plus(
                     A						,				return_attention_mask=A						,				return_token_type_ids=A						,				return_offsets_mapping=A						,				add_special_tokens=A						,				)

                 a__      =       tokenizer_r.do_lower_case if hasattr(A						,				"""do_lower_case"""    ) else False
                 a__      =       (
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), """A"""),
                         ((1, 2), ""","""),
                         ((3, 5), """na"""),
                         ((5, 6), """##ï"""),
                         ((6, 8), """##ve"""),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), """Allen"""),
                         ((21, 23), """##NL"""),
                         ((23, 24), """##P"""),
                         ((25, 33), """sentence"""),
                         ((33, 34), """."""),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), """a"""),
                         ((1, 2), ""","""),
                         ((3, 8), """naive"""),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), """allen"""),
                         ((21, 23), """##nl"""),
                         ((23, 24), """##p"""),
                         ((25, 33), """sentence"""),
                         ((33, 34), """."""),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]						,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
                 self.assertEqual([e[0] for e in expected_results]						,				tokens["""offset_mapping"""]    )






     @slow
     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			int:






         """simple docstring"""
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
                 a__      =       self.rust_tokenizer_class.from_pretrained(A						,				**A    )
                 a__      =       self.tokenizer_class.from_pretrained(A						,				**A    )
                 a__      =       [["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

                 a__      =       tokenizer_r.batch_encode_candidates(A						,				max_length=10						,				return_tensors="""np"""    )
                 a__      =       tokenizer_p.batch_encode_candidates(A						,				max_length=10						,				return_tensors="""np"""    )

                 a__      =       (2, 2, 10)

                 self.assertEqual(encoded_sentence_r["""input_ids"""].shape						,				A    )
                 self.assertEqual(encoded_sentence_r["""attention_mask"""].shape						,				A    )
                 self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape						,				A    )

                 self.assertEqual(encoded_sentence_p["""input_ids"""].shape						,				A    )
                 self.assertEqual(encoded_sentence_p["""attention_mask"""].shape						,				A    )
                 self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape						,				A    )












import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


SCREAMING_SNAKE_CASE_						:					Optional[Any]			   =       logging.get_logger(__name__)

if is_tf_available():
 import tensorflow as tf







@dataclass
class 		__UpperCAmelCase(		lowercase__ ):





     """simple docstring"""



     __UpperCAmelCase	: int						=    """tf"""
     __UpperCAmelCase	: Optional[str]						=    field(
         default=lowercase__       ,	metadata={"""help""": """Name of TPU"""}       ,	)

     __UpperCAmelCase	: Optional[str]						=    field(
         default=lowercase__       ,	metadata={"""help""": """Zone of TPU"""}       ,	)

     __UpperCAmelCase	: Optional[str]						=    field(
         default=lowercase__       ,	metadata={"""help""": """Name of Cloud TPU-enabled project"""}       ,	)

     __UpperCAmelCase	: float						=    field(
         default=1.0       ,	metadata={"""help""": """Power for the Polynomial decay LR scheduler."""}       ,	)

     __UpperCAmelCase	: bool						=    field(default=lowercase__       ,	metadata={"""help""": """Whether to activate the XLA compilation or not"""} )

     @cached_property
     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict    )			->			Tuple["tf.distribute.Strategy", int]:






         """simple docstring"""
         requires_backends(self						,				["""tf"""]    )
         logger.info("""Tensorflow: setting up strategy"""    )

         a__      =       tf.config.list_physical_devices("""GPU"""    )

         # Set to float16 at first
         if self.fpaa:
             tf.keras.mixed_precision.set_global_policy("""mixed_float16"""    )

         if self.no_cuda:
             a__      =       tf.distribute.OneDeviceStrategy(device="""/cpu:0"""    )
         else:
             try:
                 if self.tpu_name:
                     a__      =       tf.distribute.cluster_resolver.TPUClusterResolver(
                         self.tpu_name						,				zone=self.tpu_zone						,				project=self.gcp_project    )
                 else:
                     a__      =       tf.distribute.cluster_resolver.TPUClusterResolver()
             except ValueError:
                 if self.tpu_name:
                     raise RuntimeError(f"""Couldn't connect to TPU {self.tpu_name}!"""    )
                 else:
                     a__      =       None

             if tpu:
                 # Set to bfloat16 in case of TPU
                 if self.fpaa:
                     tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""    )

                 tf.config.experimental_connect_to_cluster(A    )
                 tf.tpu.experimental.initialize_tpu_system(A    )

                 a__      =       tf.distribute.TPUStrategy(A    )

             elif len(A    ) == 0:
                 a__      =       tf.distribute.OneDeviceStrategy(device="""/cpu:0"""    )
             elif len(A    ) == 1:
                 a__      =       tf.distribute.OneDeviceStrategy(device="""/gpu:0"""    )
             elif len(A    ) > 1:
                 # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
                 a__      =       tf.distribute.MirroredStrategy()
             else:
                 raise ValueError("""Cannot find the proper strategy, please check your environment properties."""    )

         return strategy

     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			"tf.distribute.Strategy":






         """simple docstring"""
         requires_backends(self						,				["""tf"""]    )
         return self._setup_strategy

     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict    )			->			int:






         """simple docstring"""
         requires_backends(self						,				["""tf"""]    )
         return self._setup_strategy.num_replicas_in_sync

     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict    )			->			List[Any]:






         """simple docstring"""
         return False  # TF Logging is handled by Keras not the Trainer

     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[Any]    )			->			int:






         """simple docstring"""
         if self.per_gpu_train_batch_size:
             logger.warning(
                 """Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future """
                 """version. Using `--per_device_train_batch_size` is preferred."""    )
         a__      =       self.per_gpu_train_batch_size or self.per_device_train_batch_size
         return per_device_batch_size * self.n_replicas

     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]    )			->			int:






         """simple docstring"""
         if self.per_gpu_eval_batch_size:
             logger.warning(
                 """Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future """
                 """version. Using `--per_device_eval_batch_size` is preferred."""    )
         a__      =       self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
         return per_device_batch_size * self.n_replicas






     @property
     def 				__SCREAMING_SNAKE_CASE		(      self :       str    )			->			int:






         """simple docstring"""
         requires_backends(self						,				["""tf"""]    )
         warnings.warn(
             """The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead."""						,				A						,				)
         return self._setup_strategy.num_replicas_in_sync






