

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__					(		lowerCamelCase__						,	unittest.TestCase   ):







						"""simple docstring"""




						lowercase  :				Optional[Any]											=							PreTrainedTokenizerFast
						lowercase  :				Optional[int]											=							False
						lowercase  :				Union[str, Any]											=							True
						lowercase  :				Union[str, Any]											=							'tokenizer_file'

						def __lowerCamelCase					(				self    )							->	Tuple:




									'''simple docstring'''


									__UpperCamelCase    :    int								=      False  # because we don't have pretrained_vocab_files_map
									super().setUp()
									__UpperCamelCase    :    Optional[Any]								=      True

									__UpperCamelCase    :    Union[str, Any]								=      ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
									__UpperCamelCase    :    Dict								=      "SaulLu/dummy-tokenizer-bytelevel-bpe"

									# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
									__UpperCamelCase    :    Dict								=      [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

									__UpperCamelCase    :    Any								=      PreTrainedTokenizerFast.from_pretrained(model_paths[0]    )
									tokenizer.save_pretrained(self.tmpdirname    )

						def __lowerCamelCase					(				self    )							->	Union[str, Any]:




									'''simple docstring'''


									pass

						def __lowerCamelCase					(				self    )							->	str:




									'''simple docstring'''


									pass

						def __lowerCamelCase					(				self    )							->	Union[str, Any]:




									'''simple docstring'''


									pass

						def __lowerCamelCase					(				self    )							->	Dict:




									'''simple docstring'''


									pass

						def __lowerCamelCase					(				self    )							->	Optional[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Any								=      self.tmpdirname
									# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															try:
																		__UpperCamelCase    :    int								=      tempfile.mkdtemp()
																		__UpperCamelCase    :    int								=      self.rust_tokenizer_class.from_pretrained(__UpperCamelCase , **__UpperCamelCase    )

																		tokenizer.save_pretrained(self.tmpdirname    )
																		super().test_training_new_tokenizer()
															finally:
																		# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																		# is restored
																		shutil.rmtree(self.tmpdirname    )
																		__UpperCamelCase    :    Tuple								=      tmpdirname_orig

						def __lowerCamelCase					(				self    )							->	Dict:




									'''simple docstring'''


									__UpperCamelCase    :    Dict								=      self.tmpdirname
									# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															try:
																		__UpperCamelCase    :    Union[str, Any]								=      tempfile.mkdtemp()
																		__UpperCamelCase    :    str								=      self.rust_tokenizer_class.from_pretrained(__UpperCamelCase , **__UpperCamelCase    )

																		tokenizer.save_pretrained(self.tmpdirname    )
																		super().test_training_new_tokenizer_with_special_tokens_change()
															finally:
																		# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																		# is restored
																		shutil.rmtree(self.tmpdirname    )
																		__UpperCamelCase    :    Any								=      tmpdirname_orig




						def __lowerCamelCase					(				self    )							->	Optional[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Dict								=      self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name    )

									__UpperCamelCase    :    List[str]								=      ("a" for _ in range(10_00    ))
									__UpperCamelCase    :    Optional[int]								=      tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase , length=10_00 , vocab_size=50    )

									__UpperCamelCase    :    List[Any]								=      new_tokenizer.encode("a🤗"    )
									self.assertEqual(__UpperCamelCase , [64, 1_72, 2_53, 97, 2_45]    )







@require_tokenizers
class 		SCREAMING_SNAKE_CASE__					(		unittest.TestCase   ):







						"""simple docstring"""




						def __lowerCamelCase					(				self    )							->	Any:




									'''simple docstring'''


									__UpperCamelCase    :    Optional[Any]								=      AutoTokenizer.from_pretrained("bert-base-cased"    )
									__UpperCamelCase    :    Tuple								=      json.loads(tokenizer._tokenizer.to_str()    )
									__UpperCamelCase    :    int								=      len(__UpperCamelCase    )

									with tempfile.TemporaryDirectory() as tmp_dir:
												# Hack to save this in the tokenizer_config.json
												__UpperCamelCase    :    Optional[int]								=      ["tokenizer.4.0.0.json"]
												tokenizer.save_pretrained(__UpperCamelCase    )
												json.dump(__UpperCamelCase , open(os.path.join(__UpperCamelCase , "tokenizer.4.0.0.json"    ) , "w"    )    )

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												__UpperCamelCase    :    List[str]								=      AutoTokenizer.from_pretrained(__UpperCamelCase    )
												self.assertEqual(len(__UpperCamelCase    ) , len(__UpperCamelCase    ) + 1    )
												__UpperCamelCase    :    Optional[Any]								=      json.loads(new_tokenizer._tokenizer.to_str()    )
												self.assertIn("huggingface" , json_tokenizer["model"]["vocab"]    )

												# Will need to be adjusted if we reach v42 and this test is still here.
												# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
												shutil.move(os.path.join(__UpperCamelCase , "tokenizer.4.0.0.json"    ) , os.path.join(__UpperCamelCase , "tokenizer.42.0.0.json"    )    )
												__UpperCamelCase    :    Dict								=      ["tokenizer.42.0.0.json"]
												tokenizer.save_pretrained(__UpperCamelCase    )
												__UpperCamelCase    :    int								=      AutoTokenizer.from_pretrained(__UpperCamelCase    )
												self.assertEqual(len(__UpperCamelCase    ) , len(__UpperCamelCase    )    )
												__UpperCamelCase    :    Dict								=      json.loads(new_tokenizer._tokenizer.to_str()    )
												self.assertNotIn("huggingface" , json_tokenizer["model"]["vocab"]    )




						def __lowerCamelCase					(				self    )							->	Union[str, Any]:




									'''simple docstring'''


									__UpperCamelCase    :    List[str]								=      "hf-internal-testing/test-two-tokenizers"

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									__UpperCamelCase    :    Any								=      AutoTokenizer.from_pretrained(__UpperCamelCase    )
									self.assertEqual(len(__UpperCamelCase    ) , 2_89_97    )
									__UpperCamelCase    :    Dict								=      json.loads(tokenizer._tokenizer.to_str()    )
									self.assertIn("huggingface" , json_tokenizer["model"]["vocab"]    )

									# Testing an older version by monkey-patching the version in the module it's used.
									import transformers as old_transformers

									__UpperCamelCase    :    int								=      "3.0.0"
									__UpperCamelCase    :    int								=      old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase    )
									self.assertEqual(len(__UpperCamelCase    ) , 2_89_96    )
									__UpperCamelCase    :    int								=      json.loads(old_tokenizer._tokenizer.to_str()    )
									self.assertNotIn("huggingface" , json_tokenizer["model"]["vocab"]    )







@require_tokenizers
class 		SCREAMING_SNAKE_CASE__					(		unittest.TestCase   ):







						"""simple docstring"""




						def __lowerCamelCase					(				self    )							->	Tuple:




									'''simple docstring'''


									__UpperCamelCase    :    Optional[int]								=      PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"    )
									__UpperCamelCase    :    Dict								=      "The Matrix is a 1999 science fiction action film."

									with concurrent.futures.ThreadPoolExecutor() as executor:
												__UpperCamelCase    :    Dict								=      [executor.submit(self.fetch , __UpperCamelCase , __UpperCamelCase    ) for i in range(10    )]
												__UpperCamelCase    :    str								=      [future.result() for future in futures]
												self.assertEqual(__UpperCamelCase , [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10    )]    )




						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase    )							->	int:




									'''simple docstring'''


									return tokenizer.encode(__UpperCamelCase , truncation="longest_first" , padding="longest"    )


import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


lowercase     :       Dict        =       "\nHugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.\n\nIn March 2021, Hugging Face raised $40 million in a Series B funding round.[3]\n\nOn April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]\n"
class 		SCREAMING_SNAKE_CASE__					(		unittest.TestCase						,	lowerCamelCase__   ):







						"""simple docstring"""




						def __lowerCamelCase					(				self    )							->	Union[str, Any]:




									'''simple docstring'''


									__UpperCamelCase    :    str								=      load_tool("summarization"    )
									self.tool.setup()
									__UpperCamelCase    :    str								=      load_tool("summarization" , remote=__UpperCamelCase    )

						def __lowerCamelCase					(				self    )							->	Optional[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Union[str, Any]								=      self.tool(__UpperCamelCase    )
									self.assertEqual(
									    __UpperCamelCase , "Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM." , )

						def __lowerCamelCase					(				self    )							->	int:




									'''simple docstring'''


									__UpperCamelCase    :    Any								=      self.remote_tool(__UpperCamelCase    )
									self.assertEqual(
									    __UpperCamelCase , "Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM." , )

						def __lowerCamelCase					(				self    )							->	List[Any]:




									'''simple docstring'''


									__UpperCamelCase    :    List[Any]								=      self.tool(text=__UpperCamelCase    )
									self.assertEqual(
									    __UpperCamelCase , "Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM." , )




						def __lowerCamelCase					(				self    )							->	str:




									'''simple docstring'''


									__UpperCamelCase    :    str								=      self.remote_tool(text=__UpperCamelCase    )
									self.assertEqual(
									    __UpperCamelCase , "Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM." , )