




"""simple docstring"""







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							snake_case_ (				__lowerCamelCase  ,   unittest.TestCase ):







   """simple docstring"""




   A_										=					PreTrainedTokenizerFast
   A_										=					False
   A_										=					True
   A_										=					'tokenizer_file'







   def      UpperCAmelCase__ (		self)							->      str:
       UpperCamelCase										=  False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       UpperCamelCase										=  True

       UpperCamelCase										=  ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       UpperCamelCase										=  '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       UpperCamelCase										=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       UpperCamelCase										=  PreTrainedTokenizerFast.from_pretrained(model_paths[0])
       tokenizer.save_pretrained(self.tmpdirname)







   def      UpperCAmelCase__ (		self)							->      Any:
       pass







   def      UpperCAmelCase__ (		self)							->      str:
       pass







   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       pass







   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       pass







   def      UpperCAmelCase__ (		self)							->      int:
       UpperCamelCase										=  self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
               try:
                   UpperCamelCase										=  tempfile.mkdtemp()
                   UpperCamelCase										=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase_					,				**lowerCamelCase_)

                   tokenizer.save_pretrained(self.tmpdirname)
                   super().test_training_new_tokenizer()
               finally:
                   # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                   # is restored
                   shutil.rmtree(self.tmpdirname)
                   UpperCamelCase										=  tmpdirname_orig







   def      UpperCAmelCase__ (		self)							->      str:
       UpperCamelCase										=  self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
               try:
                   UpperCamelCase										=  tempfile.mkdtemp()
                   UpperCamelCase										=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase_					,				**lowerCamelCase_)

                   tokenizer.save_pretrained(self.tmpdirname)
                   super().test_training_new_tokenizer_with_special_tokens_change()
               finally:
                   # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                   # is restored
                   shutil.rmtree(self.tmpdirname)
                   UpperCamelCase										=  tmpdirname_orig







   def      UpperCAmelCase__ (		self)							->      Any:
       UpperCamelCase										=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

       UpperCamelCase										=  ('''a''' for _ in range(1_0_0_0))
       UpperCamelCase										=  tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase_					,				length=1_0_0_0					,				vocab_size=5_0)

       UpperCamelCase										=  new_tokenizer.encode('''a🤗''')
       self.assertEqual(lowerCamelCase_					,				[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5])

@require_tokenizers
class 							snake_case_ (				unittest.TestCase ):







   """simple docstring"""




   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       UpperCamelCase										=  AutoTokenizer.from_pretrained('''bert-base-cased''')
       UpperCamelCase										=  json.loads(tokenizer._tokenizer.to_str())
       UpperCamelCase										=  len(lowerCamelCase_)

       with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           UpperCamelCase										=  ['''tokenizer.4.0.0.json''']
           tokenizer.save_pretrained(lowerCamelCase_)
           json.dump(lowerCamelCase_					,				open(os.path.join(lowerCamelCase_					,				'''tokenizer.4.0.0.json''')					,				'''w'''))

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           UpperCamelCase										=  AutoTokenizer.from_pretrained(lowerCamelCase_)
           self.assertEqual(len(lowerCamelCase_)					,				len(lowerCamelCase_) + 1)
           UpperCamelCase										=  json.loads(new_tokenizer._tokenizer.to_str())
           self.assertIn('''huggingface'''					,				json_tokenizer['''model''']['''vocab'''])

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(lowerCamelCase_					,				'''tokenizer.4.0.0.json''')					,				os.path.join(lowerCamelCase_					,				'''tokenizer.42.0.0.json'''))
           UpperCamelCase										=  ['''tokenizer.42.0.0.json''']
           tokenizer.save_pretrained(lowerCamelCase_)
           UpperCamelCase										=  AutoTokenizer.from_pretrained(lowerCamelCase_)
           self.assertEqual(len(lowerCamelCase_)					,				len(lowerCamelCase_))
           UpperCamelCase										=  json.loads(new_tokenizer._tokenizer.to_str())
           self.assertNotIn('''huggingface'''					,				json_tokenizer['''model''']['''vocab'''])







   def      UpperCAmelCase__ (		self)							->      Tuple:
       UpperCamelCase										=  '''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       UpperCamelCase										=  AutoTokenizer.from_pretrained(lowerCamelCase_)
       self.assertEqual(len(lowerCamelCase_)					,				2_8_9_9_7)
       UpperCamelCase										=  json.loads(tokenizer._tokenizer.to_str())
       self.assertIn('''huggingface'''					,				json_tokenizer['''model''']['''vocab'''])

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       UpperCamelCase										=  '''3.0.0'''
       UpperCamelCase										=  old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase_)
       self.assertEqual(len(lowerCamelCase_)					,				2_8_9_9_6)
       UpperCamelCase										=  json.loads(old_tokenizer._tokenizer.to_str())
       self.assertNotIn('''huggingface'''					,				json_tokenizer['''model''']['''vocab'''])

@require_tokenizers
class 							snake_case_ (				unittest.TestCase ):







   """simple docstring"""




   def      UpperCAmelCase__ (		self)							->      Any:
       UpperCamelCase										=  PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel''')
       UpperCamelCase										=  '''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
           UpperCamelCase										=  [executor.submit(self.fetch					,				lowerCamelCase_					,				lowerCamelCase_) for i in range(1_0)]
           UpperCamelCase										=  [future.result() for future in futures]
           self.assertEqual(lowerCamelCase_					,				[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0)])







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_)							->      Optional[int]:
       return tokenizer.encode(lowerCamelCase_					,				truncation='''longest_first'''					,				padding='''longest''')





"""simple docstring"""







from math import pi





def     __snake_case   (							_lowercase						):


    """simple docstring"""







    return degree / (180 / pi)


if __name__ == "__main__":
  from doctest import testmod

  testmod()