


"""simple docstring"""





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class  lowercase_  ( a_							,							unittest.TestCase						):
      __magic_name__     :							str          =				RealmTokenizer
      __magic_name__     :							Optional[Any]          =				RealmTokenizerFast
      __magic_name__     :							Any          =				True
      __magic_name__     :							Tuple          =				True
      __magic_name__     :							Tuple          =				filter_non_english







      def 				_lowerCAmelCase       (      self :							Tuple    ):
        super().setUp()

        lowerCAmelCase__    :    str								=					[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "[PAD]",
            "[MASK]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        lowerCAmelCase__    :    str								=					os.path.join(self.tmpdirname		,  VOCAB_FILES_NAMES["vocab_file"]    )
        with open(self.vocab_file		,  "w"		,  encoding="utf-8"    ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )







      def 				_lowerCAmelCase       (      self :							Optional[Any]		,  _lowercase :							Union[str, Any]    ):
        lowerCAmelCase__    :    str								=					"UNwant\u00E9d,running"
        lowerCAmelCase__    :    List[str]								=					"unwanted, running"
        return input_text, output_text







      def 				_lowerCAmelCase       (      self :							Union[str, Any]    ):
        lowerCAmelCase__    :    List[Any]								=					self.tokenizer_class(self.vocab_file    )

        lowerCAmelCase__    :    Any								=					tokenizer.tokenize("UNwant\u00E9d,running"    )
        self.assertListEqual(_lowercase		,  ["un", "##want", "##ed", ",", "runn", "##ing"]    )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase    )		,  [9, 6, 7, 1_2, 1_0, 1_1]    )







      def 				_lowerCAmelCase       (      self :							Any    ):
        if not self.test_rust_tokenizer:
          return

        lowerCAmelCase__    :    Optional[int]								=					self.get_tokenizer()
        lowerCAmelCase__    :    Any								=					self.get_rust_tokenizer()

        lowerCAmelCase__    :    Any								=					"UNwant\u00E9d,running"

        lowerCAmelCase__    :    Dict								=					tokenizer.tokenize(_lowercase    )
        lowerCAmelCase__    :    Dict								=					rust_tokenizer.tokenize(_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )

        lowerCAmelCase__    :    Tuple								=					tokenizer.encode(_lowercase		,  add_special_tokens=_lowercase    )
        lowerCAmelCase__    :    Optional[int]								=					rust_tokenizer.encode(_lowercase		,  add_special_tokens=_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )

        lowerCAmelCase__    :    Optional[Any]								=					self.get_rust_tokenizer()
        lowerCAmelCase__    :    int								=					tokenizer.encode(_lowercase    )
        lowerCAmelCase__    :    List[Any]								=					rust_tokenizer.encode(_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )

        # With lower casing
        lowerCAmelCase__    :    Tuple								=					self.get_tokenizer(do_lower_case=_lowercase    )
        lowerCAmelCase__    :    str								=					self.get_rust_tokenizer(do_lower_case=_lowercase    )

        lowerCAmelCase__    :    Tuple								=					"UNwant\u00E9d,running"

        lowerCAmelCase__    :    Dict								=					tokenizer.tokenize(_lowercase    )
        lowerCAmelCase__    :    Tuple								=					rust_tokenizer.tokenize(_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )

        lowerCAmelCase__    :    Optional[int]								=					tokenizer.encode(_lowercase		,  add_special_tokens=_lowercase    )
        lowerCAmelCase__    :    Union[str, Any]								=					rust_tokenizer.encode(_lowercase		,  add_special_tokens=_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )

        lowerCAmelCase__    :    List[str]								=					self.get_rust_tokenizer()
        lowerCAmelCase__    :    int								=					tokenizer.encode(_lowercase    )
        lowerCAmelCase__    :    List[Any]								=					rust_tokenizer.encode(_lowercase    )
        self.assertListEqual(_lowercase		,  _lowercase    )







      def 				_lowerCAmelCase       (      self :							int    ):
        lowerCAmelCase__    :    Any								=					BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"    )		,  ["ah", "\u535A", "\u63A8", "zz"]    )







      def 				_lowerCAmelCase       (      self :							Optional[int]    ):
        lowerCAmelCase__    :    int								=					BasicTokenizer(do_lower_case=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )		,  ["hello", "!", "how", "are", "you", "?"]    )
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )		,  ["hello"]    )







      def 				_lowerCAmelCase       (      self :							List[Any]    ):
        lowerCAmelCase__    :    Optional[Any]								=					BasicTokenizer(do_lower_case=_lowercase		,  strip_accents=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )		,  ["hällo", "!", "how", "are", "you", "?"]    )
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )		,  ["h\u00E9llo"]    )







      def 				_lowerCAmelCase       (      self :							Any    ):
        lowerCAmelCase__    :    List[Any]								=					BasicTokenizer(do_lower_case=_lowercase		,  strip_accents=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )		,  ["hallo", "!", "how", "are", "you", "?"]    )
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )		,  ["hello"]    )







      def 				_lowerCAmelCase       (      self :							Tuple    ):
        lowerCAmelCase__    :    Optional[Any]								=					BasicTokenizer(do_lower_case=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )		,  ["hallo", "!", "how", "are", "you", "?"]    )
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )		,  ["hello"]    )







      def 				_lowerCAmelCase       (      self :							int    ):
        lowerCAmelCase__    :    Any								=					BasicTokenizer(do_lower_case=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )		,  ["HeLLo", "!", "how", "Are", "yoU", "?"]    )







      def 				_lowerCAmelCase       (      self :							Optional[Any]    ):
        lowerCAmelCase__    :    Optional[Any]								=					BasicTokenizer(do_lower_case=_lowercase		,  strip_accents=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )		,  ["HäLLo", "!", "how", "Are", "yoU", "?"]    )







      def 				_lowerCAmelCase       (      self :							Any    ):
        lowerCAmelCase__    :    Optional[int]								=					BasicTokenizer(do_lower_case=_lowercase		,  strip_accents=_lowercase    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )		,  ["HaLLo", "!", "how", "Are", "yoU", "?"]    )







      def 				_lowerCAmelCase       (      self :							List[str]    ):
        lowerCAmelCase__    :    int								=					BasicTokenizer(do_lower_case=_lowercase		,  never_split=["[UNK]"]    )

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"    )		,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]    )







      def 				_lowerCAmelCase       (      self :							Tuple    ):
        lowerCAmelCase__    :    Any								=					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

        lowerCAmelCase__    :    Any								=					{}
        for i, token in enumerate(_lowercase    ):
          lowerCAmelCase__    :    Optional[Any]								=					i
        lowerCAmelCase__    :    Tuple								=					WordpieceTokenizer(vocab=_lowercase		,  unk_token="[UNK]"    )

        self.assertListEqual(tokenizer.tokenize(""    )		,  []    )

        self.assertListEqual(tokenizer.tokenize("unwanted running"    )		,  ["un", "##want", "##ed", "runn", "##ing"]    )

        self.assertListEqual(tokenizer.tokenize("unwantedX running"    )		,  ["[UNK]", "runn", "##ing"]    )







      def 				_lowerCAmelCase       (      self :							Optional[Any]    ):
        self.assertTrue(_is_whitespace(" "    )    )
        self.assertTrue(_is_whitespace("\t"    )    )
        self.assertTrue(_is_whitespace("\r"    )    )
        self.assertTrue(_is_whitespace("\n"    )    )
        self.assertTrue(_is_whitespace("\u00A0"    )    )

        self.assertFalse(_is_whitespace("A"    )    )
        self.assertFalse(_is_whitespace("-"    )    )







      def 				_lowerCAmelCase       (      self :							List[str]    ):
        self.assertTrue(_is_control("\u0005"    )    )

        self.assertFalse(_is_control("A"    )    )
        self.assertFalse(_is_control(" "    )    )
        self.assertFalse(_is_control("\t"    )    )
        self.assertFalse(_is_control("\r"    )    )







      def 				_lowerCAmelCase       (      self :							Union[str, Any]    ):
        self.assertTrue(_is_punctuation("-"    )    )
        self.assertTrue(_is_punctuation("$"    )    )
        self.assertTrue(_is_punctuation("`"    )    )
        self.assertTrue(_is_punctuation("."    )    )

        self.assertFalse(_is_punctuation("A"    )    )
        self.assertFalse(_is_punctuation(" "    )    )







      def 				_lowerCAmelCase       (      self :							Optional[int]    ):
        lowerCAmelCase__    :    Tuple								=					self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(_lowercase    ) for t in ["Test", "\xad", "test"]]		,  [["[UNK]"], [], ["[UNK]"]]    )

        if self.test_rust_tokenizer:
          lowerCAmelCase__    :    Optional[int]								=					self.get_rust_tokenizer()
          self.assertListEqual(
              [rust_tokenizer.tokenize(_lowercase    ) for t in ["Test", "\xad", "test"]]		,  [["[UNK]"], [], ["[UNK]"]]    )







      @slow
      def 				_lowerCAmelCase       (      self :							Tuple    ):
        lowerCAmelCase__    :    List[str]								=					self.tokenizer_class.from_pretrained("bert-base-uncased"    )

        lowerCAmelCase__    :    Dict								=					tokenizer.encode("sequence builders"		,  add_special_tokens=_lowercase    )
        lowerCAmelCase__    :    List[str]								=					tokenizer.encode("multi-sequence build"		,  add_special_tokens=_lowercase    )

        lowerCAmelCase__    :    Union[str, Any]								=					tokenizer.build_inputs_with_special_tokens(_lowercase    )
        lowerCAmelCase__    :    Union[str, Any]								=					tokenizer.build_inputs_with_special_tokens(_lowercase		,  _lowercase    )

        assert encoded_sentence == [1_0_1] + text + [1_0_2]
        assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







      def 				_lowerCAmelCase       (      self :							Optional[int]    ):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
            lowerCAmelCase__    :    List[str]								=					self.rust_tokenizer_class.from_pretrained(_lowercase		,  **_lowercase    )

            lowerCAmelCase__    :    Dict								=					f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
            lowerCAmelCase__    :    Dict								=					tokenizer_r.encode_plus(
                _lowercase		,  return_attention_mask=_lowercase		,  return_token_type_ids=_lowercase		,  return_offsets_mapping=_lowercase		,  add_special_tokens=_lowercase		,  )

            lowerCAmelCase__    :    Union[str, Any]								=					tokenizer_r.do_lower_case if hasattr(_lowercase		,  "do_lower_case"    ) else False
            lowerCAmelCase__    :    int								=					(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "A"),
                    ((1, 2), ","),
                    ((3, 5), "na"),
                    ((5, 6), "##ï"),
                    ((6, 8), "##ve"),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), "Allen"),
                    ((2_1, 2_3), "##NL"),
                    ((2_3, 2_4), "##P"),
                    ((2_5, 3_3), "sentence"),
                    ((3_3, 3_4), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "a"),
                    ((1, 2), ","),
                    ((3, 8), "naive"),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), "allen"),
                    ((2_1, 2_3), "##nl"),
                    ((2_3, 2_4), "##p"),
                    ((2_5, 3_3), "sentence"),
                    ((3_3, 3_4), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]		,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]    )    )
            self.assertEqual([e[0] for e in expected_results]		,  tokens["offset_mapping"]    )







      @slow
      def 				_lowerCAmelCase       (      self :							Optional[int]    ):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
            lowerCAmelCase__    :    Optional[int]								=					self.rust_tokenizer_class.from_pretrained(_lowercase		,  **_lowercase    )
            lowerCAmelCase__    :    Optional[int]								=					self.tokenizer_class.from_pretrained(_lowercase		,  **_lowercase    )
            lowerCAmelCase__    :    str								=					[["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

            lowerCAmelCase__    :    List[str]								=					tokenizer_r.batch_encode_candidates(_lowercase		,  max_length=1_0		,  return_tensors="np"    )
            lowerCAmelCase__    :    str								=					tokenizer_p.batch_encode_candidates(_lowercase		,  max_length=1_0		,  return_tensors="np"    )

            lowerCAmelCase__    :    Union[str, Any]								=					(2, 2, 1_0)

            self.assertEqual(encoded_sentence_r["input_ids"].shape		,  _lowercase    )
            self.assertEqual(encoded_sentence_r["attention_mask"].shape		,  _lowercase    )
            self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,  _lowercase    )

            self.assertEqual(encoded_sentence_p["input_ids"].shape		,  _lowercase    )
            self.assertEqual(encoded_sentence_p["attention_mask"].shape		,  _lowercase    )
            self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,  _lowercase    )




"""simple docstring"""





def     lowercase__	(  lowerCamelCase     :  int				)					->     int:
  if not isinstance(lowerCamelCase				,	lowerCamelCase				):
    raise TypeError("recursive_lucas_number accepts only integer arguments."				)
  if n_th_number == 0:
    return 2
  if n_th_number == 1:
    return 1

  return recursive_lucas_number(n_th_number - 1				) + recursive_lucas_number(
      n_th_number - 2				)





def     lowercase__	(  lowerCamelCase     :  int				)					->     int:
  if not isinstance(lowerCamelCase				,	lowerCamelCase				):
    raise TypeError("dynamic_lucas_number accepts only integer arguments."				)
  lowerCAmelCase__				,		lowerCAmelCase__    :    str								=					2, 1
  for _ in range(lowerCamelCase				):
    lowerCAmelCase__				,		lowerCAmelCase__    :    str								=					b, a + b
  return a


if __name__ == "__main__":
   from doctest import testmod

   testmod()
   __UpperCAmelCase    				=      int(input("Enter the number of terms in lucas series:\n").strip())
   print("Using recursive function to calculate lucas series:")
   print(" ".join(str(recursive_lucas_number(i)) for i in range(n)))
   print("\nUsing dynamic function to calculate lucas series:")
   print(" ".join(str(dynamic_lucas_number(i)) for i in range(n)))
