




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 				_a   (a__,		unittest.TestCase  ):





     '''simple docstring'''



     lowerCAmelCase_     :    Optional[Any]									=				PreTrainedTokenizerFast
     lowerCAmelCase_     :    List[Any]									=				False
     lowerCAmelCase_     :    Tuple									=				True
     lowerCAmelCase_     :    Dict									=				"""tokenizer_file"""







     def  snake_case_			(							self       )			->							List[str]:
          snake_case :			Optional[Any]					   = False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          snake_case :			List[Any]					   = True

          snake_case :			Dict					   = ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
          snake_case :			Tuple					   = """SaulLu/dummy-tokenizer-bytelevel-bpe"""

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          snake_case :			int					   = [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          snake_case :			int					   = PreTrainedTokenizerFast.from_pretrained(model_paths[0]       )
          tokenizer.save_pretrained(self.tmpdirname       )







     def  snake_case_			(							self       )			->							List[str]:
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







     def  snake_case_			(							self       )			->							List[Any]:
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







     def  snake_case_			(							self       )			->							Optional[Any]:
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







     def  snake_case_			(							self       )			->							Optional[int]:
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass







     def  snake_case_			(							self       )			->							Tuple:
          snake_case :			Any					   = self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                    try:
                         snake_case :			Tuple					   = tempfile.mkdtemp()
                         snake_case :			Any					   = self.rust_tokenizer_class.from_pretrained(__a       ,**__a       )

                         tokenizer.save_pretrained(self.tmpdirname       )
                         super().test_training_new_tokenizer()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname       )
                         snake_case :			Tuple					   = tmpdirname_orig







     def  snake_case_			(							self       )			->							List[Any]:
          snake_case :			List[Any]					   = self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                    try:
                         snake_case :			Tuple					   = tempfile.mkdtemp()
                         snake_case :			Dict					   = self.rust_tokenizer_class.from_pretrained(__a       ,**__a       )

                         tokenizer.save_pretrained(self.tmpdirname       )
                         super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname       )
                         snake_case :			Union[str, Any]					   = tmpdirname_orig







     def  snake_case_			(							self       )			->							List[Any]:
          snake_case :			Optional[Any]					   = self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name       )

          snake_case :			Optional[Any]					   = ("""a""" for _ in range(1_000       ))
          snake_case :			int					   = tokenizer.train_new_from_iterator(text_iterator=__a       ,length=1_000       ,vocab_size=50       )

          snake_case :			Tuple					   = new_tokenizer.encode("""a🤗"""       )
          self.assertEqual(__a       ,[64, 172, 253, 97, 245]       )







@require_tokenizers
class 				_a   (unittest.TestCase  ):





     '''simple docstring'''



     def  snake_case_			(							self       )			->							Tuple:
          snake_case :			Dict					   = AutoTokenizer.from_pretrained("""bert-base-cased"""       )
          snake_case :			Tuple					   = json.loads(tokenizer._tokenizer.to_str()       )
          snake_case :			Dict					   = len(__a       )

          with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               snake_case :			Optional[int]					   = ["""tokenizer.4.0.0.json"""]
               tokenizer.save_pretrained(__a       )
               json.dump(__a       ,open(os.path.join(__a       ,"""tokenizer.4.0.0.json"""       )       ,"""w"""       )       )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               snake_case :			Optional[int]					   = AutoTokenizer.from_pretrained(__a       )
               self.assertEqual(len(__a       )       ,len(__a       ) + 1       )
               snake_case :			List[str]					   = json.loads(new_tokenizer._tokenizer.to_str()       )
               self.assertIn("""huggingface"""       ,json_tokenizer["""model"""]["""vocab"""]       )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(__a       ,"""tokenizer.4.0.0.json"""       )       ,os.path.join(__a       ,"""tokenizer.42.0.0.json"""       )       )
               snake_case :			Any					   = ["""tokenizer.42.0.0.json"""]
               tokenizer.save_pretrained(__a       )
               snake_case :			Union[str, Any]					   = AutoTokenizer.from_pretrained(__a       )
               self.assertEqual(len(__a       )       ,len(__a       )       )
               snake_case :			List[str]					   = json.loads(new_tokenizer._tokenizer.to_str()       )
               self.assertNotIn("""huggingface"""       ,json_tokenizer["""model"""]["""vocab"""]       )







     def  snake_case_			(							self       )			->							Optional[Any]:
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          snake_case :			Dict					   = """hf-internal-testing/test-two-tokenizers"""

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          snake_case :			Tuple					   = AutoTokenizer.from_pretrained(__a       )
          self.assertEqual(len(__a       )       ,28_997       )
          snake_case :			Tuple					   = json.loads(tokenizer._tokenizer.to_str()       )
          self.assertIn("""huggingface"""       ,json_tokenizer["""model"""]["""vocab"""]       )

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          snake_case :			List[str]					   = """3.0.0"""
          snake_case :			Tuple					   = old_transformers.models.auto.AutoTokenizer.from_pretrained(__a       )
          self.assertEqual(len(__a       )       ,28_996       )
          snake_case :			List[str]					   = json.loads(old_tokenizer._tokenizer.to_str()       )
          self.assertNotIn("""huggingface"""       ,json_tokenizer["""model"""]["""vocab"""]       )







@require_tokenizers
class 				_a   (unittest.TestCase  ):





     '''simple docstring'''



     def  snake_case_			(							self       )			->							List[str]:
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          snake_case :			int					   = PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""       )
          snake_case :			int					   = """The Matrix is a 1999 science fiction action film."""

          with concurrent.futures.ThreadPoolExecutor() as executor:
               snake_case :			Any					   = [executor.submit(self.fetch       ,__a       ,__a       ) for i in range(10       )]
               snake_case :			List[Any]					   = [future.result() for future in futures]
               self.assertEqual(__a       ,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10       )]       )







     def  snake_case_			(							self       ,__a       ,__a       )			->							List[Any]:
          return tokenizer.encode(__a       ,truncation="""longest_first"""       ,padding="""longest"""       )









'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase			:      Any			=					{"""configuration_visual_bert""": ["""VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """VisualBertConfig"""]}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase			:      Optional[int]			=					[
          """VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """VisualBertForMultipleChoice""",
          """VisualBertForPreTraining""",
          """VisualBertForQuestionAnswering""",
          """VisualBertForRegionToPhraseAlignment""",
          """VisualBertForVisualReasoning""",
          """VisualBertLayer""",
          """VisualBertModel""",
          """VisualBertPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_visual_bert import (
                VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                VisualBertForMultipleChoice,
                VisualBertForPreTraining,
                VisualBertForQuestionAnswering,
                VisualBertForRegionToPhraseAlignment,
                VisualBertForVisualReasoning,
                VisualBertLayer,
                VisualBertModel,
                VisualBertPreTrainedModel,
            )


else:
      import sys

      lowercase			:      List[Any]			=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



