


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	_snake_case						(   _snake_case		,					unittest.TestCase							):
 SCREAMING_SNAKE_CASE__   = PreTrainedTokenizerFast
 SCREAMING_SNAKE_CASE__   = False
 SCREAMING_SNAKE_CASE__   = True
 SCREAMING_SNAKE_CASE__   = 'tokenizer_file'




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Union[str, Any]											= False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       a		:int											= True

       a		:Optional[int]											= ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       a		:int											= '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       a		:str											= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       a		:Optional[Any]											= PreTrainedTokenizerFast.from_pretrained(model_paths[0]					)
       tokenizer.save_pretrained(self.tmpdirname					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Union[str, Any]											= self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   try:
                         a		:Union[str, Any]											= tempfile.mkdtemp()
                         a		:Tuple											= self.rust_tokenizer_class.from_pretrained(_lowerCamelCase      ,						**_lowerCamelCase					)

                         tokenizer.save_pretrained(self.tmpdirname					)
                         super().test_training_new_tokenizer()
                   finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname					)
                         a		:List[Any]											= tmpdirname_orig




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Union[str, Any]											= self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   try:
                         a		:Dict											= tempfile.mkdtemp()
                         a		:Optional[int]											= self.rust_tokenizer_class.from_pretrained(_lowerCamelCase      ,						**_lowerCamelCase					)

                         tokenizer.save_pretrained(self.tmpdirname					)
                         super().test_training_new_tokenizer_with_special_tokens_change()
                   finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname					)
                         a		:List[Any]											= tmpdirname_orig




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:str											= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name					)

       a		:str											= ('''a''' for _ in range(1000					))
       a		:str											= tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase      ,						length=1000      ,						vocab_size=50					)

       a		:Union[str, Any]											= new_tokenizer.encode('''a🤗'''					)
       self.assertEqual(_lowerCamelCase      ,						[64, 172, 253, 97, 245]					)







@require_tokenizers
class 	_snake_case						(   unittest.TestCase							):




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Dict											= AutoTokenizer.from_pretrained('''bert-base-cased'''					)
       a		:List[Any]											= json.loads(tokenizer._tokenizer.to_str()					)
       a		:List[Any]											= len(_lowerCamelCase					)

       with tempfile.TemporaryDirectory() as tmp_dir:
             # Hack to save this in the tokenizer_config.json
             a		:int											= ['''tokenizer.4.0.0.json''']
             tokenizer.save_pretrained(_lowerCamelCase					)
             json.dump(_lowerCamelCase      ,						open(os.path.join(_lowerCamelCase      ,						'''tokenizer.4.0.0.json'''					)      ,						'''w'''					)					)

             # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
             a		:str											= AutoTokenizer.from_pretrained(_lowerCamelCase					)
             self.assertEqual(len(_lowerCamelCase					)      ,						len(_lowerCamelCase					) + 1					)
             a		:List[str]											= json.loads(new_tokenizer._tokenizer.to_str()					)
             self.assertIn('''huggingface'''      ,						json_tokenizer['''model''']['''vocab''']					)

             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(_lowerCamelCase      ,						'''tokenizer.4.0.0.json'''					)      ,						os.path.join(_lowerCamelCase      ,						'''tokenizer.42.0.0.json'''					)					)
             a		:str											= ['''tokenizer.42.0.0.json''']
             tokenizer.save_pretrained(_lowerCamelCase					)
             a		:Dict											= AutoTokenizer.from_pretrained(_lowerCamelCase					)
             self.assertEqual(len(_lowerCamelCase					)      ,						len(_lowerCamelCase					)					)
             a		:Any											= json.loads(new_tokenizer._tokenizer.to_str()					)
             self.assertNotIn('''huggingface'''      ,						json_tokenizer['''model''']['''vocab''']					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       a		:List[str]											= '''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       a		:Union[str, Any]											= AutoTokenizer.from_pretrained(_lowerCamelCase					)
       self.assertEqual(len(_lowerCamelCase					)      ,						2_8997					)
       a		:Tuple											= json.loads(tokenizer._tokenizer.to_str()					)
       self.assertIn('''huggingface'''      ,						json_tokenizer['''model''']['''vocab''']					)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       a		:List[str]											= '''3.0.0'''
       a		:Optional[Any]											= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase					)
       self.assertEqual(len(_lowerCamelCase					)      ,						2_8996					)
       a		:Optional[int]											= json.loads(old_tokenizer._tokenizer.to_str()					)
       self.assertNotIn('''huggingface'''      ,						json_tokenizer['''model''']['''vocab''']					)







@require_tokenizers
class 	_snake_case						(   unittest.TestCase							):




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       a		:List[Any]											= PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''					)
       a		:str											= '''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
             a		:List[Any]											= [executor.submit(self.fetch      ,						_lowerCamelCase      ,						_lowerCamelCase					) for i in range(10					)]
             a		:List[Any]											= [future.result() for future in futures]
             self.assertEqual(_lowerCamelCase      ,						[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10					)]					)




 def  SCREAMING_SNAKE_CASE__       (	self      ,						_lowerCamelCase      ,						_lowerCamelCase					):
       return tokenizer.encode(_lowerCamelCase      ,						truncation='''longest_first'''      ,						padding='''longest'''					)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


snake_case       :					Dict        =  {
    '''configuration_align''': [
        '''ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AlignConfig''',
        '''AlignTextConfig''',
        '''AlignVisionConfig''',
    ],
    '''processing_align''': ['''AlignProcessor'''],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     snake_case       :					List[str]        =  [
         '''ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''AlignModel''',
         '''AlignPreTrainedModel''',
         '''AlignTextModel''',
         '''AlignVisionModel''',
     ]

if TYPE_CHECKING:
     from .configuration_align import (
         ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
         AlignConfig,
         AlignTextConfig,
         AlignVisionConfig,
     )
     from .processing_align import AlignProcessor

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_align import (
              ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
              AlignModel,
              AlignPreTrainedModel,
              AlignTextModel,
              AlignVisionModel,
          )

else:
     import sys

     snake_case       :					int        =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





