




'''simple docstring'''





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     UpperCamelCase					(  __lowerCamelCase			,		unittest.TestCase					):
  a    :Dict      					=				PreTrainedTokenizerFast
  a    :List[Any]      					=				False
  a    :Any      					=				True
  a    :int      					=				'tokenizer_file'



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Optional[Any]       					=			False  # because we don't have pretrained_vocab_files_map
      super().setUp()
      _a							:       Any       					=			True

      _a							:       List[str]       					=			["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
      _a							:       Dict       					=			"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

      # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
      _a							:       str       					=			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

      _a							:       Any       					=			PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
      tokenizer.save_pretrained(self.tmpdirname   )



  def 						a_       ( self   ):







      '''simple docstring'''


      pass



  def 						a_       ( self   ):







      '''simple docstring'''


      pass



  def 						a_       ( self   ):







      '''simple docstring'''


      pass



  def 						a_       ( self   ):







      '''simple docstring'''


      pass



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       str       					=			self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
              try:
                  _a							:       Dict       					=			tempfile.mkdtemp()
                  _a							:       Union[str, Any]       					=			self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase		,    **__lowerCAmelCase   )

                  tokenizer.save_pretrained(self.tmpdirname   )
                  super().test_training_new_tokenizer()
              finally:
                  # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                  # is restored
                  shutil.rmtree(self.tmpdirname   )
                  _a							:       List[str]       					=			tmpdirname_orig



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Dict       					=			self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
              try:
                  _a							:       Optional[Any]       					=			tempfile.mkdtemp()
                  _a							:       Dict       					=			self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase		,    **__lowerCAmelCase   )

                  tokenizer.save_pretrained(self.tmpdirname   )
                  super().test_training_new_tokenizer_with_special_tokens_change()
              finally:
                  # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                  # is restored
                  shutil.rmtree(self.tmpdirname   )
                  _a							:       Dict       					=			tmpdirname_orig



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       List[str]       					=			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

      _a							:       int       					=			("""a""" for _ in range(1000   ))
      _a							:       Tuple       					=			tokenizer.train_new_from_iterator(text_iterator=__lowerCAmelCase		,    length=1000		,    vocab_size=50   )

      _a							:       Optional[Any]       					=			new_tokenizer.encode("""a🤗"""   )
      self.assertEqual(__lowerCAmelCase		,    [64, 172, 253, 97, 245]   )






@require_tokenizers
class     UpperCamelCase					(  unittest.TestCase					):



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       List[Any]       					=			AutoTokenizer.from_pretrained("""bert-base-cased"""   )
      _a							:       List[Any]       					=			json.loads(tokenizer._tokenizer.to_str()   )
      _a							:       List[str]       					=			len(__lowerCAmelCase   )

      with tempfile.TemporaryDirectory() as tmp_dir:
          # Hack to save this in the tokenizer_config.json
          _a							:       List[Any]       					=			["""tokenizer.4.0.0.json"""]
          tokenizer.save_pretrained(__lowerCAmelCase   )
          json.dump(__lowerCAmelCase		,    open(os.path.join(__lowerCAmelCase		,    """tokenizer.4.0.0.json"""   )		,    """w"""   )   )

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          _a							:       str       					=			AutoTokenizer.from_pretrained(__lowerCAmelCase   )
          self.assertEqual(len(__lowerCAmelCase   )		,    len(__lowerCAmelCase   ) + 1   )
          _a							:       Optional[int]       					=			json.loads(new_tokenizer._tokenizer.to_str()   )
          self.assertIn("""huggingface"""		,    json_tokenizer["""model"""]["""vocab"""]   )

          # Will need to be adjusted if we reach v42 and this test is still here.
          # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
          shutil.move(os.path.join(__lowerCAmelCase		,    """tokenizer.4.0.0.json"""   )		,    os.path.join(__lowerCAmelCase		,    """tokenizer.42.0.0.json"""   )   )
          _a							:       Union[str, Any]       					=			["""tokenizer.42.0.0.json"""]
          tokenizer.save_pretrained(__lowerCAmelCase   )
          _a							:       str       					=			AutoTokenizer.from_pretrained(__lowerCAmelCase   )
          self.assertEqual(len(__lowerCAmelCase   )		,    len(__lowerCAmelCase   )   )
          _a							:       str       					=			json.loads(new_tokenizer._tokenizer.to_str()   )
          self.assertNotIn("""huggingface"""		,    json_tokenizer["""model"""]["""vocab"""]   )



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       str       					=			"""hf-internal-testing/test-two-tokenizers"""

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      _a							:       Dict       					=			AutoTokenizer.from_pretrained(__lowerCAmelCase   )
      self.assertEqual(len(__lowerCAmelCase   )		,    2_8997   )
      _a							:       Tuple       					=			json.loads(tokenizer._tokenizer.to_str()   )
      self.assertIn("""huggingface"""		,    json_tokenizer["""model"""]["""vocab"""]   )

      # Testing an older version by monkey-patching the version in the module it's used.
      import transformers as old_transformers

      _a							:       List[Any]       					=			"""3.0.0"""
      _a							:       Tuple       					=			old_transformers.models.auto.AutoTokenizer.from_pretrained(__lowerCAmelCase   )
      self.assertEqual(len(__lowerCAmelCase   )		,    2_8996   )
      _a							:       Optional[Any]       					=			json.loads(old_tokenizer._tokenizer.to_str()   )
      self.assertNotIn("""huggingface"""		,    json_tokenizer["""model"""]["""vocab"""]   )






@require_tokenizers
class     UpperCamelCase					(  unittest.TestCase					):



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Optional[Any]       					=			PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""   )
      _a							:       Dict       					=			"""The Matrix is a 1999 science fiction action film."""

      with concurrent.futures.ThreadPoolExecutor() as executor:
          _a							:       Dict       					=			[executor.submit(self.fetch		,    __lowerCAmelCase		,    __lowerCAmelCase   ) for i in range(10   )]
          _a							:       Optional[Any]       					=			[future.result() for future in futures]
          self.assertEqual(__lowerCAmelCase		,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10   )]   )



  def 						a_       ( self		,    __lowerCAmelCase		,    __lowerCAmelCase   ):







      '''simple docstring'''


      return tokenizer.encode(__lowerCAmelCase		,    truncation="""longest_first"""		,    padding="""longest"""   )








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


a		:  Dict   			=			{
    '''configuration_mobilevitv2''': [
        '''MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''MobileViTV2Config''',
        '''MobileViTV2OnnxConfig''',
    ],
}


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a		:  List[str]   			=			[
      '''MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''MobileViTV2ForImageClassification''',
      '''MobileViTV2ForSemanticSegmentation''',
      '''MobileViTV2Model''',
      '''MobileViTV2PreTrainedModel''',
  ]

if TYPE_CHECKING:
  from .configuration_mobilevitva import (
      MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
      MobileViTVaConfig,
      MobileViTVaOnnxConfig,
  )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_mobilevitva import (
        MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
        MobileViTVaForImageClassification,
        MobileViTVaForSemanticSegmentation,
        MobileViTVaModel,
        MobileViTVaPreTrainedModel,
    )

else:
  import sys

  a		:  Dict   			=			_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


