


'''simple docstring'''


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					lowercase__						(		_snake_case  ,  unittest.TestCase      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						PreTrainedTokenizerFast
							A_							:   Tuple   			=						False
							A_							:   Dict   			=						True
							A_							:   Optional[Any]   			=						"""tokenizer_file"""






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: int  	=   False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											_SCREAMING_SNAKE_CASE				: Tuple  	=   True

											_SCREAMING_SNAKE_CASE				: List[Any]  	=   ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   """SaulLu/dummy-tokenizer-bytelevel-bpe"""

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
											tokenizer.save_pretrained(self.tmpdirname						)






							def UpperCAmelCase_			(					self						):
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass






							def UpperCAmelCase_			(					self						):
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass






							def UpperCAmelCase_			(					self						):
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass






							def UpperCAmelCase_			(					self						):
											# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
											pass






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
																			try:
																							_SCREAMING_SNAKE_CASE				: List[Any]  	=   tempfile.mkdtemp()
																							_SCREAMING_SNAKE_CASE				: str  	=   self.rust_tokenizer_class.from_pretrained(__snake_case					,      **__snake_case						)

																							tokenizer.save_pretrained(self.tmpdirname						)
																							super().test_training_new_tokenizer()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname						)
																							_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   tmpdirname_orig






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: str  	=   self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
																			try:
																							_SCREAMING_SNAKE_CASE				: Any  	=   tempfile.mkdtemp()
																							_SCREAMING_SNAKE_CASE				: Dict  	=   self.rust_tokenizer_class.from_pretrained(__snake_case					,      **__snake_case						)

																							tokenizer.save_pretrained(self.tmpdirname						)
																							super().test_training_new_tokenizer_with_special_tokens_change()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname						)
																							_SCREAMING_SNAKE_CASE				: int  	=   tmpdirname_orig






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Any  	=   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

											_SCREAMING_SNAKE_CASE				: List[Any]  	=   ("""a""" for _ in range(1000						))
											_SCREAMING_SNAKE_CASE				: int  	=   tokenizer.train_new_from_iterator(text_iterator=__snake_case					,      length=1000					,      vocab_size=50						)

											_SCREAMING_SNAKE_CASE				: int  	=   new_tokenizer.encode("""a🤗"""						)
											self.assertEqual(__snake_case					,      [64, 172, 253, 97, 245]						)





@require_tokenizers
class 					lowercase__						(		unittest.TestCase      ):





							'''simple docstring'''


							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   AutoTokenizer.from_pretrained("""bert-base-cased"""						)
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   json.loads(tokenizer._tokenizer.to_str()						)
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   len(__snake_case						)

											with tempfile.TemporaryDirectory() as tmp_dir:
															# Hack to save this in the tokenizer_config.json
															_SCREAMING_SNAKE_CASE				: Dict  	=   ["""tokenizer.4.0.0.json"""]
															tokenizer.save_pretrained(__snake_case						)
															json.dump(__snake_case					,      open(os.path.join(__snake_case					,      """tokenizer.4.0.0.json"""						)					,      """w"""						)						)

															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
															_SCREAMING_SNAKE_CASE				: Any  	=   AutoTokenizer.from_pretrained(__snake_case						)
															self.assertEqual(len(__snake_case						)					,      len(__snake_case						) + 1						)
															_SCREAMING_SNAKE_CASE				: List[str]  	=   json.loads(new_tokenizer._tokenizer.to_str()						)
															self.assertIn("""huggingface"""					,      json_tokenizer["""model"""]["""vocab"""]						)

															# Will need to be adjusted if we reach v42 and this test is still here.
															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
															shutil.move(os.path.join(__snake_case					,      """tokenizer.4.0.0.json"""						)					,      os.path.join(__snake_case					,      """tokenizer.42.0.0.json"""						)						)
															_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   ["""tokenizer.42.0.0.json"""]
															tokenizer.save_pretrained(__snake_case						)
															_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   AutoTokenizer.from_pretrained(__snake_case						)
															self.assertEqual(len(__snake_case						)					,      len(__snake_case						)						)
															_SCREAMING_SNAKE_CASE				: Optional[int]  	=   json.loads(new_tokenizer._tokenizer.to_str()						)
															self.assertNotIn("""huggingface"""					,      json_tokenizer["""model"""]["""vocab"""]						)






							def UpperCAmelCase_			(					self						):
											# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   """hf-internal-testing/test-two-tokenizers"""

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											_SCREAMING_SNAKE_CASE				: List[str]  	=   AutoTokenizer.from_pretrained(__snake_case						)
											self.assertEqual(len(__snake_case						)					,      2_8997						)
											_SCREAMING_SNAKE_CASE				: str  	=   json.loads(tokenizer._tokenizer.to_str()						)
											self.assertIn("""huggingface"""					,      json_tokenizer["""model"""]["""vocab"""]						)

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											_SCREAMING_SNAKE_CASE				: Tuple  	=   """3.0.0"""
											_SCREAMING_SNAKE_CASE				: int  	=   old_transformers.models.auto.AutoTokenizer.from_pretrained(__snake_case						)
											self.assertEqual(len(__snake_case						)					,      2_8996						)
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   json.loads(old_tokenizer._tokenizer.to_str()						)
											self.assertNotIn("""huggingface"""					,      json_tokenizer["""model"""]["""vocab"""]						)





@require_tokenizers
class 					lowercase__						(		unittest.TestCase      ):





							'''simple docstring'''


							def UpperCAmelCase_			(					self						):
											# See https://github.com/huggingface/transformers/pull/12550
											# and https://github.com/huggingface/tokenizers/issues/537
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""						)
											_SCREAMING_SNAKE_CASE				: str  	=   """The Matrix is a 1999 science fiction action film."""

											with concurrent.futures.ThreadPoolExecutor() as executor:
															_SCREAMING_SNAKE_CASE				: str  	=   [executor.submit(self.fetch					,      __snake_case					,      __snake_case						) for i in range(10						)]
															_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   [future.result() for future in futures]
															self.assertEqual(__snake_case					,      [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10						)]						)






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case						):
											return tokenizer.encode(__snake_case					,      truncation="""longest_first"""					,      padding="""longest"""						)








'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase_	:     Any							     =	{
    'configuration_musicgen': [
        'MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MusicgenConfig',
        'MusicgenDecoderConfig',
    ],
    'processing_musicgen': ['MusicgenProcessor'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCAmelCase_	:     Any							     =	[
				    'MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'MusicgenForConditionalGeneration',
				    'MusicgenForCausalLM',
				    'MusicgenModel',
				    'MusicgenPreTrainedModel',
				]

if TYPE_CHECKING:
				from .configuration_musicgen import (
				    MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP,
				    MusicgenConfig,
				    MusicgenDecoderConfig,
				)
				from .processing_musicgen import MusicgenProcessor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_musicgen import (
								    MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST,
								    MusicgenForCausalLM,
								    MusicgenForConditionalGeneration,
								    MusicgenModel,
								    MusicgenPreTrainedModel,
								)

else:
				import sys

				UpperCAmelCase_	:     Tuple							     =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




