





'''simple docstring'''


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class    a_     (       __lowerCamelCase			,		unittest.TestCase							):
     __A  				=    PreTrainedTokenizerFast
     __A  				=    False
     __A  				=    True
     __A  				=    "tokenizer_file"





     def 							lowercase__   (				self	:      str		):





        """simple docstring"""




        lowercase_    :Union[str, Any]					    =				False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        lowercase_    :Dict					    =				True

        lowercase_    :int					    =				["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
        lowercase_    :List[Any]					    =				"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        lowercase_    :int					    =				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        lowercase_    :Union[str, Any]					    =				PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
        tokenizer.save_pretrained(self.tmpdirname		)





     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        pass





     def 							lowercase__   (				self	:      int		):





        """simple docstring"""




        pass





     def 							lowercase__   (				self	:      int		):





        """simple docstring"""




        pass





     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        pass





     def 							lowercase__   (				self	:      int		):





        """simple docstring"""




        lowercase_    :Optional[int]					    =				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
              try:
                 lowercase_    :str					    =				tempfile.mkdtemp()
                 lowercase_    :Any					    =				self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase			,   **_lowerCAmelCase		)

                 tokenizer.save_pretrained(self.tmpdirname		)
                 super().test_training_new_tokenizer()
              finally:
                 # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                 # is restored
                 shutil.rmtree(self.tmpdirname		)
                 lowercase_    :str					    =				tmpdirname_orig





     def 							lowercase__   (				self	:      Dict		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
              try:
                 lowercase_    :Optional[Any]					    =				tempfile.mkdtemp()
                 lowercase_    :Any					    =				self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase			,   **_lowerCAmelCase		)

                 tokenizer.save_pretrained(self.tmpdirname		)
                 super().test_training_new_tokenizer_with_special_tokens_change()
              finally:
                 # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                 # is restored
                 shutil.rmtree(self.tmpdirname		)
                 lowercase_    :str					    =				tmpdirname_orig







     def 							lowercase__   (				self	:      Dict		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)

        lowercase_    :Dict					    =				("""a""" for _ in range(1_000		))
        lowercase_    :int					    =				tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase			,   length=1_000			,   vocab_size=50		)

        lowercase_    :Optional[Any]					    =				new_tokenizer.encode("a🤗"		)
        self.assertEqual(_lowerCAmelCase			,   [64, 172, 253, 97, 245]		)







@require_tokenizers
class    a_     (       unittest.TestCase							):





     def 							lowercase__   (				self	:      Any		):





        """simple docstring"""




        lowercase_    :List[str]					    =				AutoTokenizer.from_pretrained("bert-base-cased"		)
        lowercase_    :List[Any]					    =				json.loads(tokenizer._tokenizer.to_str()		)
        lowercase_    :Optional[Any]					    =				len(_lowerCAmelCase		)

        with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           lowercase_    :Tuple					    =				["""tokenizer.4.0.0.json"""]
           tokenizer.save_pretrained(_lowerCAmelCase		)
           json.dump(_lowerCAmelCase			,   open(os.path.join(_lowerCAmelCase			,   "tokenizer.4.0.0.json"		)			,   "w"		)		)

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           lowercase_    :Optional[int]					    =				AutoTokenizer.from_pretrained(_lowerCAmelCase		)
           self.assertEqual(len(_lowerCAmelCase		)			,   len(_lowerCAmelCase		) + 1		)
           lowercase_    :Any					    =				json.loads(new_tokenizer._tokenizer.to_str()		)
           self.assertIn("huggingface"			,   json_tokenizer["model"]["vocab"]		)

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(_lowerCAmelCase			,   "tokenizer.4.0.0.json"		)			,   os.path.join(_lowerCAmelCase			,   "tokenizer.42.0.0.json"		)		)
           lowercase_    :Union[str, Any]					    =				["""tokenizer.42.0.0.json"""]
           tokenizer.save_pretrained(_lowerCAmelCase		)
           lowercase_    :Optional[Any]					    =				AutoTokenizer.from_pretrained(_lowerCAmelCase		)
           self.assertEqual(len(_lowerCAmelCase		)			,   len(_lowerCAmelCase		)		)
           lowercase_    :List[Any]					    =				json.loads(new_tokenizer._tokenizer.to_str()		)
           self.assertNotIn("huggingface"			,   json_tokenizer["model"]["vocab"]		)







     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        lowercase_    :List[str]					    =				"""hf-internal-testing/test-two-tokenizers"""

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        lowercase_    :int					    =				AutoTokenizer.from_pretrained(_lowerCAmelCase		)
        self.assertEqual(len(_lowerCAmelCase		)			,   28_997		)
        lowercase_    :List[Any]					    =				json.loads(tokenizer._tokenizer.to_str()		)
        self.assertIn("huggingface"			,   json_tokenizer["model"]["vocab"]		)

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        lowercase_    :Any					    =				"""3.0.0"""
        lowercase_    :Optional[int]					    =				old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
        self.assertEqual(len(_lowerCAmelCase		)			,   28_996		)
        lowercase_    :Optional[int]					    =				json.loads(old_tokenizer._tokenizer.to_str()		)
        self.assertNotIn("huggingface"			,   json_tokenizer["model"]["vocab"]		)







@require_tokenizers
class    a_     (       unittest.TestCase							):





     def 							lowercase__   (				self	:      Tuple		):





        """simple docstring"""




        lowercase_    :int					    =				PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"		)
        lowercase_    :List[str]					    =				"""The Matrix is a 1999 science fiction action film."""

        with concurrent.futures.ThreadPoolExecutor() as executor:
           lowercase_    :List[str]					    =				[executor.submit(self.fetch			,   _lowerCAmelCase			,   _lowerCAmelCase		) for i in range(10		)]
           lowercase_    :Dict					    =				[future.result() for future in futures]
           self.assertEqual(_lowerCAmelCase			,   [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)







     def 							lowercase__   (				self	:      Dict			,   lowercase	:      Tuple			,   lowercase	:      Union[str, Any]		):





        """simple docstring"""




        return tokenizer.encode(_lowerCAmelCase			,   truncation="longest_first"			,   padding="longest"		)









'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import EfficientNetImageProcessor
class    a_     (       unittest.TestCase							):





     def __init__(				self	:      int			,   lowercase	:      Optional[int]			,   lowercase	:      Tuple=13			,   lowercase	:      str=3			,   lowercase	:      List[str]=18			,   lowercase	:      Union[str, Any]=30			,   lowercase	:      Any=400			,   lowercase	:      Tuple=True			,   lowercase	:      Union[str, Any]=None			,   lowercase	:      Optional[Any]=True			,   lowercase	:      Optional[int]=[0.5, 0.5, 0.5]			,   lowercase	:      List[str]=[0.5, 0.5, 0.5]			,   ):





        """simple docstring"""




        lowercase_    :Dict					    =				size if size is not None else {"height": 18, "width": 18}
        lowercase_    :Optional[Any]					    =				parent
        lowercase_    :str					    =				batch_size
        lowercase_    :Optional[int]					    =				num_channels
        lowercase_    :str					    =				image_size
        lowercase_    :Tuple					    =				min_resolution
        lowercase_    :Optional[Any]					    =				max_resolution
        lowercase_    :List[str]					    =				do_resize
        lowercase_    :Optional[Any]					    =				size
        lowercase_    :Optional[int]					    =				do_normalize
        lowercase_    :Union[str, Any]					    =				image_mean
        lowercase_    :Dict					    =				image_std







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
        }







@require_torch
@require_vision
class    a_     (       _lowerCAmelCase			,		unittest.TestCase							):
     __A  				=    EfficientNetImageProcessor if is_vision_available() else None





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				EfficientNetImageProcessorTester(self		)





     @property
     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        return self.image_processor_tester.prepare_image_processor_dict()





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :Optional[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        self.assertTrue(hasattr(lowercase			,   "image_mean"		)		)
        self.assertTrue(hasattr(lowercase			,   "image_std"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_normalize"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_resize"		)		)
        self.assertTrue(hasattr(lowercase			,   "size"		)		)





     def 							lowercase__   (				self	:      str		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.image_processing_class.from_dict(self.image_processor_dict		)
        self.assertEqual(image_processor.size			,   {"height": 18, "width": 18}		)

        lowercase_    :List[str]					    =				self.image_processing_class.from_dict(self.image_processor_dict			,   size=42		)
        self.assertEqual(image_processor.size			,   {"height": 42, "width": 42}		)





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PIL images
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   Image.Image		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[Any]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :int					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random numpy tensors
        lowercase_    :List[Any]					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   numpify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   np.ndarray		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Any					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :str					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PyTorch tensors
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   torchify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   torch.Tensor		)

        # Test not batched input
        lowercase_    :str					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[int]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )


