

"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class        SCREAMING_SNAKE_CASE      (	a_					,							unittest.TestCase ):






					"""simple docstring"""



					lowercase__       				= PreTrainedTokenizerFast
					lowercase__       				= False
					lowercase__       				= True
					lowercase__       				= "tokenizer_file"







					def  __lowerCAmelCase				( self    :       str	):
						lowerCAmelCase__			:		str						       =			False  # because we don't have pretrained_vocab_files_map
						super().setUp()
						lowerCAmelCase__			:		str						       =			True

						lowerCAmelCase__			:		Optional[Any]						       =			['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
						lowerCAmelCase__			:		Tuple						       =			'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

						# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
						lowerCAmelCase__			:		Any						       =			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

						lowerCAmelCase__			:		Tuple						       =			PreTrainedTokenizerFast.from_pretrained(model_paths[0]	)
						tokenizer.save_pretrained(self.tmpdirname	)







					def  __lowerCAmelCase				( self    :       str	):
						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
						# model
						pass







					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
						# model
						pass







					def  __lowerCAmelCase				( self    :       List[Any]	):
						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
						# model
						pass







					def  __lowerCAmelCase				( self    :       Union[str, Any]	):
						# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
						pass







					def  __lowerCAmelCase				( self    :       Dict	):
						lowerCAmelCase__			:		Union[str, Any]						       =			self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
								try:
									lowerCAmelCase__			:		Optional[int]						       =			tempfile.mkdtemp()
									lowerCAmelCase__			:		List[str]						       =			self.rust_tokenizer_class.from_pretrained(lowercase_     ,**lowercase_	)

									tokenizer.save_pretrained(self.tmpdirname	)
									super().test_training_new_tokenizer()
								finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname	)
									lowerCAmelCase__			:		Any						       =			tmpdirname_orig







					def  __lowerCAmelCase				( self    :       Any	):
						lowerCAmelCase__			:		Tuple						       =			self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
								try:
									lowerCAmelCase__			:		List[str]						       =			tempfile.mkdtemp()
									lowerCAmelCase__			:		Optional[Any]						       =			self.rust_tokenizer_class.from_pretrained(lowercase_     ,**lowercase_	)

									tokenizer.save_pretrained(self.tmpdirname	)
									super().test_training_new_tokenizer_with_special_tokens_change()
								finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname	)
									lowerCAmelCase__			:		Tuple						       =			tmpdirname_orig







					def  __lowerCAmelCase				( self    :       Optional[int]	):
						lowerCAmelCase__			:		Any						       =			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name	)

						lowerCAmelCase__			:		str						       =			('''a''' for _ in range(1_0_0_0	))
						lowerCAmelCase__			:		Dict						       =			tokenizer.train_new_from_iterator(text_iterator=lowercase_     ,length=1_0_0_0     ,vocab_size=5_0	)

						lowerCAmelCase__			:		List[str]						       =			new_tokenizer.encode('''a🤗'''	)
						self.assertEqual(lowercase_     ,[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]	)







@require_tokenizers
class        SCREAMING_SNAKE_CASE      (	unittest.TestCase ):






					"""simple docstring"""



					def  __lowerCAmelCase				( self    :       Union[str, Any]	):
						lowerCAmelCase__			:		Optional[Any]						       =			AutoTokenizer.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		Any						       =			json.loads(tokenizer._tokenizer.to_str()	)
						lowerCAmelCase__			:		List[str]						       =			len(lowercase_	)

						with tempfile.TemporaryDirectory() as tmp_dir:
							# Hack to save this in the tokenizer_config.json
							lowerCAmelCase__			:		Dict						       =			['''tokenizer.4.0.0.json''']
							tokenizer.save_pretrained(lowercase_	)
							json.dump(lowercase_     ,open(os.path.join(lowercase_     ,'''tokenizer.4.0.0.json'''	)     ,'''w'''	)	)

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							lowerCAmelCase__			:		Tuple						       =			AutoTokenizer.from_pretrained(lowercase_	)
							self.assertEqual(len(lowercase_	)     ,len(lowercase_	) + 1	)
							lowerCAmelCase__			:		Optional[Any]						       =			json.loads(new_tokenizer._tokenizer.to_str()	)
							self.assertIn('''huggingface'''     ,json_tokenizer['''model''']['''vocab''']	)

							# Will need to be adjusted if we reach v42 and this test is still here.
							# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
							shutil.move(os.path.join(lowercase_     ,'''tokenizer.4.0.0.json'''	)     ,os.path.join(lowercase_     ,'''tokenizer.42.0.0.json'''	)	)
							lowerCAmelCase__			:		Union[str, Any]						       =			['''tokenizer.42.0.0.json''']
							tokenizer.save_pretrained(lowercase_	)
							lowerCAmelCase__			:		Optional[int]						       =			AutoTokenizer.from_pretrained(lowercase_	)
							self.assertEqual(len(lowercase_	)     ,len(lowercase_	)	)
							lowerCAmelCase__			:		Tuple						       =			json.loads(new_tokenizer._tokenizer.to_str()	)
							self.assertNotIn('''huggingface'''     ,json_tokenizer['''model''']['''vocab''']	)







					def  __lowerCAmelCase				( self    :       List[str]	):
						# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
						lowerCAmelCase__			:		List[str]						       =			'''hf-internal-testing/test-two-tokenizers'''

						# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
						lowerCAmelCase__			:		List[str]						       =			AutoTokenizer.from_pretrained(lowercase_	)
						self.assertEqual(len(lowercase_	)     ,2_8_9_9_7	)
						lowerCAmelCase__			:		Union[str, Any]						       =			json.loads(tokenizer._tokenizer.to_str()	)
						self.assertIn('''huggingface'''     ,json_tokenizer['''model''']['''vocab''']	)

						# Testing an older version by monkey-patching the version in the module it's used.
						import transformers as old_transformers

						lowerCAmelCase__			:		Optional[Any]						       =			'''3.0.0'''
						lowerCAmelCase__			:		str						       =			old_transformers.models.auto.AutoTokenizer.from_pretrained(lowercase_	)
						self.assertEqual(len(lowercase_	)     ,2_8_9_9_6	)
						lowerCAmelCase__			:		Dict						       =			json.loads(old_tokenizer._tokenizer.to_str()	)
						self.assertNotIn('''huggingface'''     ,json_tokenizer['''model''']['''vocab''']	)







@require_tokenizers
class        SCREAMING_SNAKE_CASE      (	unittest.TestCase ):






					"""simple docstring"""



					def  __lowerCAmelCase				( self    :       List[Any]	):
						# See https://github.com/huggingface/transformers/pull/12550
						# and https://github.com/huggingface/tokenizers/issues/537
						lowerCAmelCase__			:		Tuple						       =			PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''	)
						lowerCAmelCase__			:		int						       =			'''The Matrix is a 1999 science fiction action film.'''

						with concurrent.futures.ThreadPoolExecutor() as executor:
							lowerCAmelCase__			:		int						       =			[executor.submit(self.fetch     ,lowercase_     ,lowercase_	) for i in range(1_0	)]
							lowerCAmelCase__			:		Optional[int]						       =			[future.result() for future in futures]
							self.assertEqual(lowercase_     ,[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0	)]	)







					def  __lowerCAmelCase				( self    :       str     ,lowercase_    :       Dict     ,lowercase_    :       Dict	):
						return tokenizer.encode(lowercase_     ,truncation='''longest_first'''     ,padding='''longest'''	)







"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__UpperCamelCase						:   List[Any]							   =	logging.get_logger(__name__)







class        SCREAMING_SNAKE_CASE      (	a_ ):






					"""simple docstring"""



					lowercase__       				= ["pixel_values"]







					def __init__( self    :       Tuple     ,lowercase_    :       bool = True     ,lowercase_    :       Optional[Dict[str, int]] = None     ,lowercase_    :       PILImageResampling = PILImageResampling.BILINEAR     ,lowercase_    :       bool = True     ,lowercase_    :       Union[int, float] = 1 / 2_5_5     ,lowercase_    :       bool = True     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,**lowercase_    :       Tuple     ,):
						super().__init__(**lowercase_	)
						lowerCAmelCase__			:		Any						       =			size if size is not None else {'''height''': 2_2_4, '''width''': 2_2_4}
						lowerCAmelCase__			:		Tuple						       =			get_size_dict(lowercase_	)
						lowerCAmelCase__			:		Tuple						       =			do_resize
						lowerCAmelCase__			:		Tuple						       =			do_rescale
						lowerCAmelCase__			:		Union[str, Any]						       =			do_normalize
						lowerCAmelCase__			:		Optional[int]						       =			size
						lowerCAmelCase__			:		List[Any]						       =			resample
						lowerCAmelCase__			:		int						       =			rescale_factor
						lowerCAmelCase__			:		int						       =			image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						lowerCAmelCase__			:		Any						       =			image_std if image_std is not None else IMAGENET_STANDARD_STD







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       np.ndarray     ,lowercase_    :       Dict[str, int]     ,lowercase_    :       PILImageResampling = PILImageResampling.BILINEAR     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       str     ,):
						lowerCAmelCase__			:		Union[str, Any]						       =			get_size_dict(lowercase_	)
						if "height" not in size or "width" not in size:
							raise ValueError(F'The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}'	)
						return resize(
						    lowercase_     ,size=(size['''height'''], size['''width'''])     ,resample=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       np.ndarray     ,lowercase_    :       float     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       Tuple	):
						return rescale(lowercase_     ,scale=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       Dict     ,lowercase_    :       np.ndarray     ,lowercase_    :       Union[float, List[float]]     ,lowercase_    :       Union[float, List[float]]     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       Union[str, Any]     ,):
						return normalize(lowercase_     ,mean=lowercase_     ,std=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       ImageInput     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Dict[str, int] = None     ,lowercase_    :       PILImageResampling = None     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Optional[float] = None     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[str, TensorType]] = None     ,lowercase_    :       Union[str, ChannelDimension] = ChannelDimension.FIRST     ,**lowercase_    :       List[Any]     ,):
						lowerCAmelCase__			:		Optional[int]						       =			do_resize if do_resize is not None else self.do_resize
						lowerCAmelCase__			:		Any						       =			do_rescale if do_rescale is not None else self.do_rescale
						lowerCAmelCase__			:		Dict						       =			do_normalize if do_normalize is not None else self.do_normalize
						lowerCAmelCase__			:		Dict						       =			resample if resample is not None else self.resample
						lowerCAmelCase__			:		Dict						       =			rescale_factor if rescale_factor is not None else self.rescale_factor
						lowerCAmelCase__			:		Optional[Any]						       =			image_mean if image_mean is not None else self.image_mean
						lowerCAmelCase__			:		str						       =			image_std if image_std is not None else self.image_std

						lowerCAmelCase__			:		Any						       =			size if size is not None else self.size
						lowerCAmelCase__			:		Dict						       =			get_size_dict(lowercase_	)

						lowerCAmelCase__			:		Union[str, Any]						       =			make_list_of_images(lowercase_	)

						if not valid_images(lowercase_	):
							raise ValueError(
							    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
							    '''torch.Tensor, tf.Tensor or jax.ndarray.'''	)

						if do_resize and size is None:
							raise ValueError('''Size must be specified if do_resize is True.'''	)

						if do_rescale and rescale_factor is None:
							raise ValueError('''Rescale factor must be specified if do_rescale is True.'''	)

						# All transformations expect numpy arrays.
						lowerCAmelCase__			:		Dict						       =			[to_numpy_array(lowercase_	) for image in images]

						if do_resize:
							lowerCAmelCase__			:		Optional[Any]						       =			[self.resize(image=lowercase_     ,size=lowercase_     ,resample=lowercase_	) for image in images]

						if do_rescale:
							lowerCAmelCase__			:		List[Any]						       =			[self.rescale(image=lowercase_     ,scale=lowercase_	) for image in images]

						if do_normalize:
							lowerCAmelCase__			:		Dict						       =			[self.normalize(image=lowercase_     ,mean=lowercase_     ,std=lowercase_	) for image in images]

						lowerCAmelCase__			:		Tuple						       =			[to_channel_dimension_format(lowercase_     ,lowercase_	) for image in images]

						lowerCAmelCase__			:		int						       =			{'''pixel_values''': images}
						return BatchFeature(data=lowercase_     ,tensor_type=lowercase_	)




