



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__lowerCamelCase							= {
    """agents""": ["""Agent""", """AzureOpenAiAgent""", """HfAgent""", """LocalAgent""", """OpenAiAgent"""],
    """base""": ["""PipelineTool""", """RemoteTool""", """Tool""", """launch_gradio_demo""", """load_tool"""],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							= ["""DocumentQuestionAnsweringTool"""]
   __lowerCamelCase							= ["""ImageCaptioningTool"""]
   __lowerCamelCase							= ["""ImageQuestionAnsweringTool"""]
   __lowerCamelCase							= ["""ImageSegmentationTool"""]
   __lowerCamelCase							= ["""SpeechToTextTool"""]
   __lowerCamelCase							= ["""TextClassificationTool"""]
   __lowerCamelCase							= ["""TextQuestionAnsweringTool"""]
   __lowerCamelCase							= ["""TextSummarizationTool"""]
   __lowerCamelCase							= ["""TextToSpeechTool"""]
   __lowerCamelCase							= ["""TranslationTool"""]

if TYPE_CHECKING:
   from .agents import Agent, AzureOpenAiAgent, HfAgent, LocalAgent, OpenAiAgent
   from .base import PipelineTool, RemoteTool, Tool, launch_gradio_demo, load_tool

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .document_question_answering import DocumentQuestionAnsweringTool
      from .image_captioning import ImageCaptioningTool
      from .image_question_answering import ImageQuestionAnsweringTool
      from .image_segmentation import ImageSegmentationTool
      from .speech_to_text import SpeechToTextTool
      from .text_classification import TextClassificationTool
      from .text_question_answering import TextQuestionAnsweringTool
      from .text_summarization import TextSummarizationTool
      from .text_to_speech import TextToSpeechTool
      from .translation import TranslationTool
else:
   import sys

   __lowerCamelCase							= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCAmelCase	(		A_ ,unittest.TestCase				):
   A__      :	Dict		  =  PreTrainedTokenizerFast
   A__      :	str		  =  False
   A__      :	List[Any]		  =  True
   A__      :	List[str]		  =  "tokenizer_file"



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          snake_case :     Optional[int]           =   True

          snake_case :     List[str]           =   ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
          snake_case :     Optional[Any]           =   "SaulLu/dummy-tokenizer-bytelevel-bpe"

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          snake_case :     int           =   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          snake_case :     List[Any]           =   PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
          tokenizer.save_pretrained(self.tmpdirname				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			Union[str, Any]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			List[str]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Any           =   tempfile.mkdtemp()
                               snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     int           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			Dict:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Union[str, Any]           =   tempfile.mkdtemp()
                               snake_case :     str           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     Dict           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      int				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

          snake_case :     int           =   ("a" for _ in range(10_00				))
          snake_case :     Optional[int]           =   tokenizer.train_new_from_iterator(text_iterator=snake_case__					,		length=10_00					,		vocab_size=50				)

          snake_case :     str           =   new_tokenizer.encode("a🤗"				)
          self.assertEqual(snake_case__					,		[64, 1_72, 2_53, 97, 2_45]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Union[str, Any]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     List[str]           =   AutoTokenizer.from_pretrained("bert-base-cased"				)
          snake_case :     Union[str, Any]           =   json.loads(tokenizer._tokenizer.to_str()				)
          snake_case :     Tuple           =   len(snake_case__				)

          with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 snake_case :     List[str]           =   ["tokenizer.4.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 json.dump(snake_case__					,		open(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		"w"				)				)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 snake_case :     Optional[Any]           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				) + 1				)
                 snake_case :     Union[str, Any]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		os.path.join(snake_case__					,		"tokenizer.42.0.0.json"				)				)
                 snake_case :     Optional[Any]           =   ["tokenizer.42.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 snake_case :     Dict           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				)				)
                 snake_case :     Optional[int]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     int           =   "hf-internal-testing/test-two-tokenizers"

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          snake_case :     int           =   AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_97				)
          snake_case :     Optional[int]           =   json.loads(tokenizer._tokenizer.to_str()				)
          self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          snake_case :     Dict           =   "3.0.0"
          snake_case :     Union[str, Any]           =   old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_96				)
          snake_case :     Optional[Any]           =   json.loads(old_tokenizer._tokenizer.to_str()				)
          self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[Any]:


          '''simple docstring'''


          snake_case :     Tuple           =   PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"				)
          snake_case :     List[Any]           =   "The Matrix is a 1999 science fiction action film."

          with concurrent.futures.ThreadPoolExecutor() as executor:
                 snake_case :     Union[str, Any]           =   [executor.submit(self.fetch					,		snake_case__					,		snake_case__				) for i in range(10				)]
                 snake_case :     Optional[Any]           =   [future.result() for future in futures]
                 self.assertEqual(snake_case__					,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      str					,		snake_case__	:      List[str]					,		snake_case__	:      str				)	->			List[str]:


          '''simple docstring'''


          return tokenizer.encode(snake_case__					,		truncation="longest_first"					,		padding="longest"				)





