


"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class  lowercase_  ( __lowerCamelCase							,							unittest.TestCase						):
      __magic_name__     :							List[Any]          =				PreTrainedTokenizerFast
      __magic_name__     :							Dict          =				False
      __magic_name__     :							Tuple          =				True
      __magic_name__     :							str          =				"""tokenizer_file"""







      def 				_lowerCAmelCase       (      self :							Union[str, Any]    ):
        lowerCAmelCase__    :    Union[str, Any]								=					False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        lowerCAmelCase__    :    Dict								=					True

        lowerCAmelCase__    :    int								=					["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
        lowerCAmelCase__    :    List[Any]								=					"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        lowerCAmelCase__    :    int								=					[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        lowerCAmelCase__    :    Union[str, Any]								=					PreTrainedTokenizerFast.from_pretrained(model_paths[0]    )
        tokenizer.save_pretrained(self.tmpdirname    )







      def 				_lowerCAmelCase       (      self :							List[Any]    ):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass







      def 				_lowerCAmelCase       (      self :							Optional[int]    ):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass







      def 				_lowerCAmelCase       (      self :							List[str]    ):
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass







      def 				_lowerCAmelCase       (      self :							str    ):
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass







      def 				_lowerCAmelCase       (      self :							str    ):
        lowerCAmelCase__    :    Optional[int]								=					self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
            try:
              lowerCAmelCase__    :    str								=					tempfile.mkdtemp()
              lowerCAmelCase__    :    Any								=					self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase		,  **_lowerCAmelCase    )

              tokenizer.save_pretrained(self.tmpdirname    )
              super().test_training_new_tokenizer()
            finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname    )
              lowerCAmelCase__    :    str								=					tmpdirname_orig







      def 				_lowerCAmelCase       (      self :							int    ):
        lowerCAmelCase__    :    Tuple								=					self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
            try:
              lowerCAmelCase__    :    Optional[Any]								=					tempfile.mkdtemp()
              lowerCAmelCase__    :    Any								=					self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase		,  **_lowerCAmelCase    )

              tokenizer.save_pretrained(self.tmpdirname    )
              super().test_training_new_tokenizer_with_special_tokens_change()
            finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname    )
              lowerCAmelCase__    :    str								=					tmpdirname_orig







      def 				_lowerCAmelCase       (      self :							Tuple    ):
        lowerCAmelCase__    :    Tuple								=					self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name    )

        lowerCAmelCase__    :    Dict								=					("""a""" for _ in range(1_0_0_0    ))
        lowerCAmelCase__    :    int								=					tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase		,  length=1_0_0_0		,  vocab_size=5_0    )

        lowerCAmelCase__    :    Optional[Any]								=					new_tokenizer.encode("a🤗"    )
        self.assertEqual(_lowerCAmelCase		,  [6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]    )

@require_tokenizers
class  lowercase_  ( unittest.TestCase						):







      def 				_lowerCAmelCase       (      self :							Dict    ):
        lowerCAmelCase__    :    List[str]								=					AutoTokenizer.from_pretrained("bert-base-cased"    )
        lowerCAmelCase__    :    List[Any]								=					json.loads(tokenizer._tokenizer.to_str()    )
        lowerCAmelCase__    :    Optional[Any]								=					len(_lowerCAmelCase    )

        with tempfile.TemporaryDirectory() as tmp_dir:
          # Hack to save this in the tokenizer_config.json
          lowerCAmelCase__    :    Tuple								=					["""tokenizer.4.0.0.json"""]
          tokenizer.save_pretrained(_lowerCAmelCase    )
          json.dump(_lowerCAmelCase		,  open(os.path.join(_lowerCAmelCase		,  "tokenizer.4.0.0.json"    )		,  "w"    )    )

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          lowerCAmelCase__    :    Optional[int]								=					AutoTokenizer.from_pretrained(_lowerCAmelCase    )
          self.assertEqual(len(_lowerCAmelCase    )		,  len(_lowerCAmelCase    ) + 1    )
          lowerCAmelCase__    :    Any								=					json.loads(new_tokenizer._tokenizer.to_str()    )
          self.assertIn("huggingface"		,  json_tokenizer["model"]["vocab"]    )

          # Will need to be adjusted if we reach v42 and this test is still here.
          # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
          shutil.move(os.path.join(_lowerCAmelCase		,  "tokenizer.4.0.0.json"    )		,  os.path.join(_lowerCAmelCase		,  "tokenizer.42.0.0.json"    )    )
          lowerCAmelCase__    :    Union[str, Any]								=					["""tokenizer.42.0.0.json"""]
          tokenizer.save_pretrained(_lowerCAmelCase    )
          lowerCAmelCase__    :    Optional[Any]								=					AutoTokenizer.from_pretrained(_lowerCAmelCase    )
          self.assertEqual(len(_lowerCAmelCase    )		,  len(_lowerCAmelCase    )    )
          lowerCAmelCase__    :    List[Any]								=					json.loads(new_tokenizer._tokenizer.to_str()    )
          self.assertNotIn("huggingface"		,  json_tokenizer["model"]["vocab"]    )







      def 				_lowerCAmelCase       (      self :							List[str]    ):
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        lowerCAmelCase__    :    List[str]								=					"""hf-internal-testing/test-two-tokenizers"""

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        lowerCAmelCase__    :    int								=					AutoTokenizer.from_pretrained(_lowerCAmelCase    )
        self.assertEqual(len(_lowerCAmelCase    )		,  2_8_9_9_7    )
        lowerCAmelCase__    :    List[Any]								=					json.loads(tokenizer._tokenizer.to_str()    )
        self.assertIn("huggingface"		,  json_tokenizer["model"]["vocab"]    )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        lowerCAmelCase__    :    Any								=					"""3.0.0"""
        lowerCAmelCase__    :    Optional[int]								=					old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase    )
        self.assertEqual(len(_lowerCAmelCase    )		,  2_8_9_9_6    )
        lowerCAmelCase__    :    Optional[int]								=					json.loads(old_tokenizer._tokenizer.to_str()    )
        self.assertNotIn("huggingface"		,  json_tokenizer["model"]["vocab"]    )

@require_tokenizers
class  lowercase_  ( unittest.TestCase						):







      def 				_lowerCAmelCase       (      self :							Any    ):
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        lowerCAmelCase__    :    int								=					PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"    )
        lowerCAmelCase__    :    List[str]								=					"""The Matrix is a 1999 science fiction action film."""

        with concurrent.futures.ThreadPoolExecutor() as executor:
          lowerCAmelCase__    :    List[str]								=					[executor.submit(self.fetch		,  _lowerCAmelCase		,  _lowerCAmelCase    ) for i in range(1_0    )]
          lowerCAmelCase__    :    Dict								=					[future.result() for future in futures]
          self.assertEqual(_lowerCAmelCase		,  [[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0    )]    )







      def 				_lowerCAmelCase       (      self :							str		,  _lowercase :							List[Any]		,  _lowercase :							List[str]    ):
        return tokenizer.encode(_lowerCAmelCase		,  truncation="longest_first"		,  padding="longest"    )




"""simple docstring"""





def     lowercase__	(  lowerCamelCase     :  int				)					->     int:
  if not isinstance(lowerCamelCase				,	lowerCamelCase				):
    raise TypeError("recursive_lucas_number accepts only integer arguments."				)
  if n_th_number == 0:
    return 2
  if n_th_number == 1:
    return 1

  return recursive_lucas_number(n_th_number - 1				) + recursive_lucas_number(
      n_th_number - 2				)





def     lowercase__	(  lowerCamelCase     :  int				)					->     int:
  if not isinstance(lowerCamelCase				,	lowerCamelCase				):
    raise TypeError("dynamic_lucas_number accepts only integer arguments."				)
  lowerCAmelCase__				,		lowerCAmelCase__    :    str								=					2, 1
  for _ in range(lowerCamelCase				):
    lowerCAmelCase__				,		lowerCAmelCase__    :    str								=					b, a + b
  return a


if __name__ == "__main__":
   from doctest import testmod

   testmod()
   __UpperCAmelCase    				=      int(input("Enter the number of terms in lucas series:\n").strip())
   print("Using recursive function to calculate lucas series:")
   print(" ".join(str(recursive_lucas_number(i)) for i in range(n)))
   print("\nUsing dynamic function to calculate lucas series:")
   print(" ".join(str(dynamic_lucas_number(i)) for i in range(n)))
