

'''simple docstring'''





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 			__magic_name__					(    unittest.TestCase   ):





			def 	__magic_name__					(	self      ,						__snake_case      )     ->				List[str]:




						'''simple docstring'''
						__a						   =list(tokenizer_class.max_model_input_sizes.keys()      )
						for model_name in sa_models[:1]:
									__a						   =tokenizer_class.from_pretrained(a_      )
									self.assertIsNotNone(a_      )
									self.assertIsInstance(a_      ,						a_      )
									self.assertIsInstance(a_      ,						a_      )

									for special_tok in tokenizer.all_special_tokens:
												self.assertIsInstance(a_      ,						a_      )
												__a						   =tokenizer.convert_tokens_to_ids(a_      )
												self.assertIsInstance(a_      ,						a_      )





			def 	__magic_name__					(	self      ,						__snake_case      ,						__snake_case = None      )     ->				Dict:




						'''simple docstring'''
						__a						   =pickle.dumps(a_      )
						self.assertIsNotNone(a_      )

						__a						   =pickle.loads(a_      )

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast      ,						be_original.is_fast      )

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings      )
						else:
									self.assertIsNone(be_restored.encodings      )

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()      ,						be_restored.values()      ):
									if equal_op:
												self.assertTrue(equal_op(a_      ,						a_      )      )
									else:
												self.assertEqual(a_      ,						a_      )





			@slow
			def 	__magic_name__					(	self      )     ->				List[Any]:




						'''simple docstring'''
						self.check_tokenizer_from_pretrained(a_      )





			def 	__magic_name__					(	self      )     ->				Union[str, Any]:




						'''simple docstring'''
						self.assertEqual(TensorType('tf'      )      ,						TensorType.TENSORFLOW      )
						self.assertEqual(TensorType('pt'      )      ,						TensorType.PYTORCH      )
						self.assertEqual(TensorType('np'      )      ,						TensorType.NUMPY      )





			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				Optional[int]:




						'''simple docstring'''
						import numpy as np

						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )
						__a						   =BertTokenizerFast.from_pretrained('bert-base-cased'      )

						# Python no tensor
						with self.subTest('BatchEncoding (Python, return_tensors=None)'      ):
									self.assert_dump_and_restore(tokenizer_p('Small example to encode'      )      )

						with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'      ):
									self.assert_dump_and_restore(
									    tokenizer_p('Small example to encode'      ,						return_tensors=TensorType.NUMPY      )      ,						np.array_equal      )

						with self.subTest('BatchEncoding (Rust, return_tensors=None)'      ):
									self.assert_dump_and_restore(tokenizer_r('Small example to encode'      )      )

						with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'      ):
									self.assert_dump_and_restore(
									    tokenizer_r('Small example to encode'      ,						return_tensors=TensorType.NUMPY      )      ,						np.array_equal      )





			@require_tf
			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				List[Any]:




						'''simple docstring'''
						import tensorflow as tf

						def tf_array_equals(__snake_case      ,						__snake_case      ):
									return tf.reduce_all(tf.equal(a_      ,						a_      )      )

						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )
						__a						   =BertTokenizerFast.from_pretrained('bert-base-cased'      )

						with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'      ):
									self.assert_dump_and_restore(
									    tokenizer_p('Small example to encode'      ,						return_tensors=TensorType.TENSORFLOW      )      ,						a_      )

						with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'      ):
									self.assert_dump_and_restore(
									    tokenizer_r('Small example to encode'      ,						return_tensors=TensorType.TENSORFLOW      )      ,						a_      )





			@require_torch
			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				List[str]:




						'''simple docstring'''
						import torch

						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )
						__a						   =BertTokenizerFast.from_pretrained('bert-base-cased'      )

						with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'      ):
									self.assert_dump_and_restore(
									    tokenizer_p('Small example to encode'      ,						return_tensors=TensorType.PYTORCH      )      ,						torch.equal      )

						with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'      ):
									self.assert_dump_and_restore(
									    tokenizer_r('Small example to encode'      ,						return_tensors=TensorType.PYTORCH      )      ,						torch.equal      )





			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				int:




						'''simple docstring'''
						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )
						__a						   =BertTokenizerFast.from_pretrained('bert-base-cased'      )

						with self.subTest('Python Tokenizer'      ):
									self.assertFalse(tokenizer_p('Small example to_encode'      ).is_fast      )

						with self.subTest('Rust Tokenizer'      ):
									self.assertTrue(tokenizer_r('Small example to_encode'      ).is_fast      )





			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				Tuple:




						'''simple docstring'''
						__a						   =BertTokenizerFast.from_pretrained('bert-base-cased'      )
						__a						   =tokenizer_r(['Test', '\xad', 'test']      ,						is_split_into_words=a_      )

						self.assertEqual(encoded.word_to_tokens(0      )      ,						TokenSpan(start=1      ,						end=2      )      )
						self.assertEqual(encoded.word_to_tokens(1      )      ,						a_      )
						self.assertEqual(encoded.word_to_tokens(2      )      ,						TokenSpan(start=2      ,						end=3      )      )





			def 	__magic_name__					(	self      )     ->				Optional[Any]:




						'''simple docstring'''
						__a						   =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}      )
						__a						   =batch.convert_to_tensors(tensor_type='np'      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(2, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(2,)      )
						# test converting the converted
						with CaptureStderr() as cs:
									__a						   =batch.convert_to_tensors(tensor_type='np'      )
						self.assertFalse(len(cs.err      )      ,						msg=f'should have no warning, but got {cs.err}'      )

						__a						   =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}      )
						__a						   =batch.convert_to_tensors(tensor_type='np'      ,						prepend_batch_axis=a_      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(1, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(1,)      )





			@require_torch
			def 	__magic_name__					(	self      )     ->				List[str]:




						'''simple docstring'''
						__a						   =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}      )
						__a						   =batch.convert_to_tensors(tensor_type='pt'      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(2, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(2,)      )
						# test converting the converted
						with CaptureStderr() as cs:
									__a						   =batch.convert_to_tensors(tensor_type='pt'      )
						self.assertFalse(len(cs.err      )      ,						msg=f'should have no warning, but got {cs.err}'      )

						__a						   =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}      )
						__a						   =batch.convert_to_tensors(tensor_type='pt'      ,						prepend_batch_axis=a_      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(1, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(1,)      )





			@require_tf
			def 	__magic_name__					(	self      )     ->				Union[str, Any]:




						'''simple docstring'''
						__a						   =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}      )
						__a						   =batch.convert_to_tensors(tensor_type='tf'      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(2, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(2,)      )
						# test converting the converted
						with CaptureStderr() as cs:
									__a						   =batch.convert_to_tensors(tensor_type='tf'      )
						self.assertFalse(len(cs.err      )      ,						msg=f'should have no warning, but got {cs.err}'      )

						__a						   =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}      )
						__a						   =batch.convert_to_tensors(tensor_type='tf'      ,						prepend_batch_axis=a_      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(1, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(1,)      )





			@require_flax
			def 	__magic_name__					(	self      )     ->				List[Any]:




						'''simple docstring'''
						__a						   =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}      )
						__a						   =batch.convert_to_tensors(tensor_type='jax'      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(2, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(2,)      )
						# test converting the converted
						with CaptureStderr() as cs:
									__a						   =batch.convert_to_tensors(tensor_type='jax'      )
						self.assertFalse(len(cs.err      )      ,						msg=f'should have no warning, but got {cs.err}'      )

						__a						   =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}      )
						__a						   =batch.convert_to_tensors(tensor_type='jax'      ,						prepend_batch_axis=a_      )
						self.assertEqual(tensor_batch['inputs'].shape      ,						(1, 3)      )
						self.assertEqual(tensor_batch['labels'].shape      ,						(1,)      )





			def 	__magic_name__					(	self      )     ->				Dict:




						'''simple docstring'''
						__a						   =[{'input_ids': np.array([0, 1, 2]      )}, {'input_ids': np.array([0, 1, 2, 3]      )}]
						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )

						__a						   =tokenizer.pad(a_      ,						padding=a_      )
						self.assertTrue(isinstance(batch['input_ids']      ,						np.ndarray      )      )
						self.assertEqual(batch['input_ids'].tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
						__a						   =tokenizer.pad(a_      ,						padding=a_      ,						return_tensors='np'      )
						self.assertTrue(isinstance(batch['input_ids']      ,						np.ndarray      )      )
						self.assertEqual(batch['input_ids'].tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )





			@require_torch
			def 	__magic_name__					(	self      )     ->				Optional[int]:




						'''simple docstring'''
						import torch

						__a						   =[{'input_ids': torch.tensor([0, 1, 2]      )}, {'input_ids': torch.tensor([0, 1, 2, 3]      )}]
						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )

						__a						   =tokenizer.pad(a_      ,						padding=a_      )
						self.assertTrue(isinstance(batch['input_ids']      ,						torch.Tensor      )      )
						self.assertEqual(batch['input_ids'].tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
						__a						   =tokenizer.pad(a_      ,						padding=a_      ,						return_tensors='pt'      )
						self.assertTrue(isinstance(batch['input_ids']      ,						torch.Tensor      )      )
						self.assertEqual(batch['input_ids'].tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )





			@require_tf
			def 	__magic_name__					(	self      )     ->				Optional[Any]:




						'''simple docstring'''
						import tensorflow as tf

						__a						   =[{'input_ids': tf.constant([0, 1, 2]      )}, {'input_ids': tf.constant([0, 1, 2, 3]      )}]
						__a						   =BertTokenizer.from_pretrained('bert-base-cased'      )

						__a						   =tokenizer.pad(a_      ,						padding=a_      )
						self.assertTrue(isinstance(batch['input_ids']      ,						tf.Tensor      )      )
						self.assertEqual(batch['input_ids'].numpy().tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
						__a						   =tokenizer.pad(a_      ,						padding=a_      ,						return_tensors='tf'      )
						self.assertTrue(isinstance(batch['input_ids']      ,						tf.Tensor      )      )
						self.assertEqual(batch['input_ids'].numpy().tolist()      ,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )





			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				Any:




						'''simple docstring'''
						__a						   =Tokenizer(WordPiece(unk_token='[UNK]'      )      )
						PreTrainedTokenizerFast(tokenizer_object=a_      )





			@require_tokenizers
			def 	__magic_name__					(	self      )     ->				List[Any]:




						'''simple docstring'''
						__a						   =Tokenizer(WordPiece(unk_token='[UNK]'      )      )
						with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(a_      ,						'tokenizer.json'      )      )
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_      ,						'tokenizer.json'      )      )









from typing import Optional

from ..utils.logging import get_logger
from .audio_classificiation import AudioClassification
from .automatic_speech_recognition import AutomaticSpeechRecognition
from .base import TaskTemplate
from .image_classification import ImageClassification
from .language_modeling import LanguageModeling
from .question_answering import QuestionAnsweringExtractive
from .summarization import Summarization
from .text_classification import TextClassification


_lowerCAmelCase      :    List[Any]						       =    [
    "AutomaticSpeechRecognition",
    "AudioClassification",
    "ImageClassification",
    "LanguageModeling",
    "QuestionAnsweringExtractive",
    "Summarization",
    "TaskTemplate",
    "TextClassification",
]

_lowerCAmelCase      :    List[str]						       =    get_logger(__name__)


_lowerCAmelCase      :    Tuple						       =    {
    AutomaticSpeechRecognition.task: AutomaticSpeechRecognition,
    AudioClassification.task: AudioClassification,
    ImageClassification.task: ImageClassification,
    LanguageModeling.task: LanguageModeling,
    QuestionAnsweringExtractive.task: QuestionAnsweringExtractive,
    Summarization.task: Summarization,
    TextClassification.task: TextClassification,
}






def  UpperCamelCase_(							_snake_case     :						dict			):




			"""simple docstring"""






			__a						   =task_template_dict.get('task'			)
			if task_name is None:
						logger.warning(F'Couldn\'t find template for task \'{task_name}\'. Available templates: {list(_snake_case			)}'			)
						return None
			__a						   =NAMEaTEMPLATE.get(_snake_case			)
			return template.from_dict(_snake_case			)






