
"""simple docstring"""







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 		UpperCAmelCase_	(__lowerCamelCase  , unittest.TestCase						):
	"""simple docstring"""




	UpperCamelCase_						:     Dict				   =    PreTrainedTokenizerFast
	UpperCamelCase_						:     List[Any]				   =    False
	UpperCamelCase_						:     Any				   =    True
	UpperCamelCase_						:     int				   =    'tokenizer_file'




	def   a      ( self       : List[Any] )->					str:



						"""simple docstring"""

						UpperCAmelCase_						:    Union[str, Any]    					=  False  # because we don't have pretrained_vocab_files_map
						super().setUp()
						UpperCAmelCase_						:    Dict    					=  True

						UpperCAmelCase_						:    str    					=  ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
						UpperCAmelCase_						:    Optional[int]    					=  """SaulLu/dummy-tokenizer-bytelevel-bpe"""

						# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
						UpperCAmelCase_						:    Union[str, Any]    					=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

						UpperCAmelCase_						:    Optional[Any]    					=  PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
						tokenizer.save_pretrained(self.tmpdirname )




	def   a      ( self       : str )->					Any:



						"""simple docstring"""

						pass




	def   a      ( self       : Dict )->					str:



						"""simple docstring"""

						pass




	def   a      ( self       : List[str] )->					Optional[Any]:



						"""simple docstring"""

						pass




	def   a      ( self       : List[str] )->					Optional[Any]:



						"""simple docstring"""

						pass




	def   a      ( self       : Union[str, Any] )->					int:



						"""simple docstring"""

						UpperCAmelCase_						:    Union[str, Any]    					=  self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
																try:
																					UpperCAmelCase_						:    str    					=  tempfile.mkdtemp()
																					UpperCAmelCase_						:    List[Any]    					=  self.rust_tokenizer_class.from_pretrained(a_     ,   **a_ )

																					tokenizer.save_pretrained(self.tmpdirname )
																					super().test_training_new_tokenizer()
																finally:
																					# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																					# is restored
																					shutil.rmtree(self.tmpdirname )
																					UpperCAmelCase_						:    str    					=  tmpdirname_orig




	def   a      ( self       : List[str] )->					str:



						"""simple docstring"""

						UpperCAmelCase_						:    Tuple    					=  self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
																try:
																					UpperCAmelCase_						:    int    					=  tempfile.mkdtemp()
																					UpperCAmelCase_						:    Optional[int]    					=  self.rust_tokenizer_class.from_pretrained(a_     ,   **a_ )

																					tokenizer.save_pretrained(self.tmpdirname )
																					super().test_training_new_tokenizer_with_special_tokens_change()
																finally:
																					# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																					# is restored
																					shutil.rmtree(self.tmpdirname )
																					UpperCAmelCase_						:    Dict    					=  tmpdirname_orig






	def   a      ( self       : str )->					Any:



						"""simple docstring"""

						UpperCAmelCase_						:    str    					=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

						UpperCAmelCase_						:    List[str]    					=  ("""a""" for _ in range(10_00 ))
						UpperCAmelCase_						:    List[Any]    					=  tokenizer.train_new_from_iterator(text_iterator=a_     ,   length=10_00     ,   vocab_size=50 )

						UpperCAmelCase_						:    Optional[int]    					=  new_tokenizer.encode("""a🤗""" )
						self.assertEqual(a_     ,   [64, 1_72, 2_53, 97, 2_45] )



@require_tokenizers
class 		UpperCAmelCase_	(unittest.TestCase						):
	"""simple docstring"""




	def   a      ( self       : Any )->					Optional[int]:



						"""simple docstring"""

						UpperCAmelCase_						:    Dict    					=  AutoTokenizer.from_pretrained("""bert-base-cased""" )
						UpperCAmelCase_						:    Union[str, Any]    					=  json.loads(tokenizer._tokenizer.to_str() )
						UpperCAmelCase_						:    str    					=  len(a_ )

						with tempfile.TemporaryDirectory() as tmp_dir:
											# Hack to save this in the tokenizer_config.json
											UpperCAmelCase_						:    List[Any]    					=  ["""tokenizer.4.0.0.json"""]
											tokenizer.save_pretrained(a_ )
											json.dump(a_     ,   open(os.path.join(a_     ,   """tokenizer.4.0.0.json""" )     ,   """w""" ) )

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											UpperCAmelCase_						:    Optional[int]    					=  AutoTokenizer.from_pretrained(a_ )
											self.assertEqual(len(a_ )     ,   len(a_ ) + 1 )
											UpperCAmelCase_						:    Optional[int]    					=  json.loads(new_tokenizer._tokenizer.to_str() )
											self.assertIn("""huggingface"""     ,   json_tokenizer["""model"""]["""vocab"""] )

											# Will need to be adjusted if we reach v42 and this test is still here.
											# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
											shutil.move(os.path.join(a_     ,   """tokenizer.4.0.0.json""" )     ,   os.path.join(a_     ,   """tokenizer.42.0.0.json""" ) )
											UpperCAmelCase_						:    Optional[int]    					=  ["""tokenizer.42.0.0.json"""]
											tokenizer.save_pretrained(a_ )
											UpperCAmelCase_						:    str    					=  AutoTokenizer.from_pretrained(a_ )
											self.assertEqual(len(a_ )     ,   len(a_ ) )
											UpperCAmelCase_						:    Any    					=  json.loads(new_tokenizer._tokenizer.to_str() )
											self.assertNotIn("""huggingface"""     ,   json_tokenizer["""model"""]["""vocab"""] )






	def   a      ( self       : Optional[Any] )->					Tuple:



						"""simple docstring"""

						UpperCAmelCase_						:    List[Any]    					=  """hf-internal-testing/test-two-tokenizers"""

						# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
						UpperCAmelCase_						:    Any    					=  AutoTokenizer.from_pretrained(a_ )
						self.assertEqual(len(a_ )     ,   2_89_97 )
						UpperCAmelCase_						:    Any    					=  json.loads(tokenizer._tokenizer.to_str() )
						self.assertIn("""huggingface"""     ,   json_tokenizer["""model"""]["""vocab"""] )

						# Testing an older version by monkey-patching the version in the module it's used.
						import transformers as old_transformers

						UpperCAmelCase_						:    List[str]    					=  """3.0.0"""
						UpperCAmelCase_						:    Tuple    					=  old_transformers.models.auto.AutoTokenizer.from_pretrained(a_ )
						self.assertEqual(len(a_ )     ,   2_89_96 )
						UpperCAmelCase_						:    int    					=  json.loads(old_tokenizer._tokenizer.to_str() )
						self.assertNotIn("""huggingface"""     ,   json_tokenizer["""model"""]["""vocab"""] )



@require_tokenizers
class 		UpperCAmelCase_	(unittest.TestCase						):
	"""simple docstring"""




	def   a      ( self       : Optional[Any] )->					Any:



						"""simple docstring"""

						UpperCAmelCase_						:    Dict    					=  PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel""" )
						UpperCAmelCase_						:    int    					=  """The Matrix is a 1999 science fiction action film."""

						with concurrent.futures.ThreadPoolExecutor() as executor:
											UpperCAmelCase_						:    Optional[int]    					=  [executor.submit(self.fetch     ,   a_     ,   a_ ) for i in range(10 )]
											UpperCAmelCase_						:    List[Any]    					=  [future.result() for future in futures]
											self.assertEqual(a_     ,   [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )






	def   a      ( self       : str     ,   a_       : int     ,   a_       : str )->					Optional[int]:



						"""simple docstring"""

						return tokenizer.encode(a_     ,   truncation="""longest_first"""     ,   padding="""longest""" )



"""simple docstring"""







import unittest

from transformers.image_processing_utils import get_size_dict



class 		UpperCAmelCase_	(unittest.TestCase						):
 """simple docstring"""




 def   a      ( self       : Union[str, Any] )->					Optional[Any]:



      """simple docstring"""

      UpperCAmelCase_						:    Any    					=  {"""wrong_key""": 2_24}
      with self.assertRaises(a_ ):
           get_size_dict(a_ )

      UpperCAmelCase_						:    Union[str, Any]    					=  {"""height""": 2_24}
      with self.assertRaises(a_ ):
           get_size_dict(a_ )

      UpperCAmelCase_						:    Union[str, Any]    					=  {"""width""": 2_24, """shortest_edge""": 2_24}
      with self.assertRaises(a_ ):
           get_size_dict(a_ )

      # Test a dict with the correct keys is returned as is
      UpperCAmelCase_						:    int    					=  {"""height""": 2_24, """width""": 2_24}
      UpperCAmelCase_						:    Dict    					=  get_size_dict(a_ )
      self.assertEqual(a_     ,   a_ )

      UpperCAmelCase_						:    Optional[int]    					=  {"""shortest_edge""": 2_24}
      UpperCAmelCase_						:    Tuple    					=  get_size_dict(a_ )
      self.assertEqual(a_     ,   {"""shortest_edge""": 2_24} )

      UpperCAmelCase_						:    str    					=  {"""longest_edge""": 2_24, """shortest_edge""": 2_24}
      UpperCAmelCase_						:    Any    					=  get_size_dict(a_ )
      self.assertEqual(a_     ,   {"""longest_edge""": 2_24, """shortest_edge""": 2_24} )

      # Test a single int value which  represents (size, size)
      UpperCAmelCase_						:    Optional[Any]    					=  get_size_dict(2_24 )
      self.assertEqual(a_     ,   {"""height""": 2_24, """width""": 2_24} )

      # Test a single int value which represents the shortest edge
      UpperCAmelCase_						:    List[str]    					=  get_size_dict(2_24     ,   default_to_square=a_ )
      self.assertEqual(a_     ,   {"""shortest_edge""": 2_24} )

      # Test a tuple of ints which represents (height, width)
      UpperCAmelCase_						:    List[Any]    					=  get_size_dict((1_50, 2_00) )
      self.assertEqual(a_     ,   {"""height""": 1_50, """width""": 2_00} )

      # Test a tuple of ints which represents (width, height)
      UpperCAmelCase_						:    List[str]    					=  get_size_dict((1_50, 2_00)     ,   height_width_order=a_ )
      self.assertEqual(a_     ,   {"""height""": 2_00, """width""": 1_50} )

      # Test an int representing the shortest edge and max_size which represents the longest edge
      UpperCAmelCase_						:    List[Any]    					=  get_size_dict(2_24     ,   max_size=2_56     ,   default_to_square=a_ )
      self.assertEqual(a_     ,   {"""shortest_edge""": 2_24, """longest_edge""": 2_56} )

      # Test int with default_to_square=True and max_size fails
      with self.assertRaises(a_ ):
           get_size_dict(2_24     ,   max_size=2_56     ,   default_to_square=a_ )

