


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
     from tokenizers import Tokenizer
     from tokenizers.models import WordPiece

class     __snake_case							(      unittest.TestCase):
     def   a (	self					:	Union[str, Any]  ,			_lowerCAmelCase					:	Union[str, Any]     )       ->				int:



           '''simple docstring'''





           __magic_name__ 		=			list(tokenizer_class.max_model_input_sizes.keys()     )
           for model_name in sa_models[:1]:
                 __magic_name__ 		=			tokenizer_class.from_pretrained(_lowerCAmelCase     )
                 self.assertIsNotNone(_lowerCAmelCase     )
                 self.assertIsInstance(_lowerCAmelCase  ,			_lowerCAmelCase     )
                 self.assertIsInstance(_lowerCAmelCase  ,			_lowerCAmelCase     )

                 for special_tok in tokenizer.all_special_tokens:
                       self.assertIsInstance(_lowerCAmelCase  ,			_lowerCAmelCase     )
                       __magic_name__ 		=			tokenizer.convert_tokens_to_ids(_lowerCAmelCase     )
                       self.assertIsInstance(_lowerCAmelCase  ,			_lowerCAmelCase     )
     def   a (	self					:	Tuple  ,			_lowerCAmelCase					:	BatchEncoding  ,			_lowerCAmelCase					:	Optional[Callable] = None     )       ->				int:



           '''simple docstring'''





           __magic_name__ 		=			pickle.dumps(_lowerCAmelCase     )
           self.assertIsNotNone(_lowerCAmelCase     )

           __magic_name__ 		=			pickle.loads(_lowerCAmelCase     )

           # Ensure is_fast is correctly restored
           self.assertEqual(be_restored.is_fast  ,			be_original.is_fast     )

           # Ensure encodings are potentially correctly restored
           if be_original.is_fast:
                 self.assertIsNotNone(be_restored.encodings     )
           else:
                 self.assertIsNone(be_restored.encodings     )

           # Ensure the keys are the same
           for original_v, restored_v in zip(be_original.values()  ,			be_restored.values()     ):
                 if equal_op:
                       self.assertTrue(equal_op(_lowerCAmelCase  ,			_lowerCAmelCase     )     )
                 else:
                       self.assertEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )
     @slow
     def   a (	self					:	Tuple     )       ->				str:



           '''simple docstring'''





           self.check_tokenizer_from_pretrained(_lowerCAmelCase     )
     def   a (	self					:	int     )       ->				Union[str, Any]:



           '''simple docstring'''





           self.assertEqual(TensorType('tf'     )  ,			TensorType.TENSORFLOW     )
           self.assertEqual(TensorType('pt'     )  ,			TensorType.PYTORCH     )
           self.assertEqual(TensorType('np'     )  ,			TensorType.NUMPY     )
     @require_tokenizers
     def   a (	self					:	Optional[Any]     )       ->				Union[str, Any]:



           '''simple docstring'''





           import numpy as np

           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )
           __magic_name__ 		=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

           # Python no tensor
           with self.subTest('BatchEncoding (Python, return_tensors=None)'     ):
                 self.assert_dump_and_restore(tokenizer_p('Small example to encode'     )     )

           with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('Small example to encode'  ,			return_tensors=TensorType.NUMPY     )  ,			np.array_equal     )

           with self.subTest('BatchEncoding (Rust, return_tensors=None)'     ):
                 self.assert_dump_and_restore(tokenizer_r('Small example to encode'     )     )

           with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('Small example to encode'  ,			return_tensors=TensorType.NUMPY     )  ,			np.array_equal     )
     @require_tf
     @require_tokenizers
     def   a (	self					:	Optional[Any]     )       ->				Tuple:



           '''simple docstring'''





           import tensorflow as tf

           def tf_array_equals(_lowerCAmelCase					:	Union[str, Any]  ,			_lowerCAmelCase					:	Union[str, Any]     ):
                 return tf.reduce_all(tf.equal(_lowerCAmelCase  ,			_lowerCAmelCase     )     )

           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )
           __magic_name__ 		=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

           with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('Small example to encode'  ,			return_tensors=TensorType.TENSORFLOW     )  ,			_lowerCAmelCase     )

           with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('Small example to encode'  ,			return_tensors=TensorType.TENSORFLOW     )  ,			_lowerCAmelCase     )
     @require_torch
     @require_tokenizers
     def   a (	self					:	Any     )       ->				Tuple:



           '''simple docstring'''





           import torch

           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )
           __magic_name__ 		=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

           with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('Small example to encode'  ,			return_tensors=TensorType.PYTORCH     )  ,			torch.equal     )

           with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('Small example to encode'  ,			return_tensors=TensorType.PYTORCH     )  ,			torch.equal     )
     @require_tokenizers
     def   a (	self					:	Optional[Any]     )       ->				Dict:



           '''simple docstring'''





           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )
           __magic_name__ 		=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

           with self.subTest('Python Tokenizer'     ):
                 self.assertFalse(tokenizer_p('Small example to_encode'     ).is_fast     )

           with self.subTest('Rust Tokenizer'     ):
                 self.assertTrue(tokenizer_r('Small example to_encode'     ).is_fast     )
     @require_tokenizers
     def   a (	self					:	int     )       ->				List[str]:



           '''simple docstring'''





           __magic_name__ 		=			BertTokenizerFast.from_pretrained('bert-base-cased'     )
           __magic_name__ 		=			tokenizer_r(['Test', '\xad', 'test']  ,			is_split_into_words=_lowerCAmelCase     )

           self.assertEqual(encoded.word_to_tokens(0     )  ,			TokenSpan(start=1  ,			end=2     )     )
           self.assertEqual(encoded.word_to_tokens(1     )  ,			_lowerCAmelCase     )
           self.assertEqual(encoded.word_to_tokens(2     )  ,			TokenSpan(start=2  ,			end=3     )     )
     def   a (	self					:	Union[str, Any]     )       ->				Union[str, Any]:



           '''simple docstring'''





           __magic_name__ 		=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='np'     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(2, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
                 __magic_name__ 		=			batch.convert_to_tensors(tensor_type='np'     )
           self.assertFalse(len(cs.err     )  ,			msg=F'''should have no warning, but got {cs.err}'''     )

           __magic_name__ 		=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='np'  ,			prepend_batch_axis=_lowerCAmelCase     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(1, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(1,)     )
     @require_torch
     def   a (	self					:	Optional[int]     )       ->				int:



           '''simple docstring'''





           __magic_name__ 		=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='pt'     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(2, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
                 __magic_name__ 		=			batch.convert_to_tensors(tensor_type='pt'     )
           self.assertFalse(len(cs.err     )  ,			msg=F'''should have no warning, but got {cs.err}'''     )

           __magic_name__ 		=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='pt'  ,			prepend_batch_axis=_lowerCAmelCase     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(1, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(1,)     )
     @require_tf
     def   a (	self					:	Optional[Any]     )       ->				Optional[Any]:



           '''simple docstring'''





           __magic_name__ 		=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='tf'     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(2, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
                 __magic_name__ 		=			batch.convert_to_tensors(tensor_type='tf'     )
           self.assertFalse(len(cs.err     )  ,			msg=F'''should have no warning, but got {cs.err}'''     )

           __magic_name__ 		=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='tf'  ,			prepend_batch_axis=_lowerCAmelCase     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(1, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(1,)     )
     @require_flax
     def   a (	self					:	Tuple     )       ->				Optional[Any]:



           '''simple docstring'''





           __magic_name__ 		=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='jax'     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(2, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
                 __magic_name__ 		=			batch.convert_to_tensors(tensor_type='jax'     )
           self.assertFalse(len(cs.err     )  ,			msg=F'''should have no warning, but got {cs.err}'''     )

           __magic_name__ 		=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
           __magic_name__ 		=			batch.convert_to_tensors(tensor_type='jax'  ,			prepend_batch_axis=_lowerCAmelCase     )
           self.assertEqual(tensor_batch['inputs'].shape  ,			(1, 3)     )
           self.assertEqual(tensor_batch['labels'].shape  ,			(1,)     )
     def   a (	self					:	str     )       ->				Any:



           '''simple docstring'''





           __magic_name__ 		=			[{'input_ids': np.array([0, 1, 2]     )}, {'input_ids': np.array([0, 1, 2, 3]     )}]
           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )

           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase     )
           self.assertTrue(isinstance(batch['input_ids']  ,			np.ndarray     )     )
           self.assertEqual(batch['input_ids'].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase  ,			return_tensors='np'     )
           self.assertTrue(isinstance(batch['input_ids']  ,			np.ndarray     )     )
           self.assertEqual(batch['input_ids'].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
     @require_torch
     def   a (	self					:	Any     )       ->				int:



           '''simple docstring'''





           import torch

           __magic_name__ 		=			[{'input_ids': torch.tensor([0, 1, 2]     )}, {'input_ids': torch.tensor([0, 1, 2, 3]     )}]
           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )

           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase     )
           self.assertTrue(isinstance(batch['input_ids']  ,			torch.Tensor     )     )
           self.assertEqual(batch['input_ids'].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase  ,			return_tensors='pt'     )
           self.assertTrue(isinstance(batch['input_ids']  ,			torch.Tensor     )     )
           self.assertEqual(batch['input_ids'].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
     @require_tf
     def   a (	self					:	Union[str, Any]     )       ->				str:



           '''simple docstring'''





           import tensorflow as tf

           __magic_name__ 		=			[{'input_ids': tf.constant([0, 1, 2]     )}, {'input_ids': tf.constant([0, 1, 2, 3]     )}]
           __magic_name__ 		=			BertTokenizer.from_pretrained('bert-base-cased'     )

           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase     )
           self.assertTrue(isinstance(batch['input_ids']  ,			tf.Tensor     )     )
           self.assertEqual(batch['input_ids'].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __magic_name__ 		=			tokenizer.pad(_lowerCAmelCase  ,			padding=_lowerCAmelCase  ,			return_tensors='tf'     )
           self.assertTrue(isinstance(batch['input_ids']  ,			tf.Tensor     )     )
           self.assertEqual(batch['input_ids'].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
     @require_tokenizers
     def   a (	self					:	int     )       ->				List[str]:



           '''simple docstring'''





           __magic_name__ 		=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
           PreTrainedTokenizerFast(tokenizer_object=_lowerCAmelCase     )







     @require_tokenizers
     def   a (	self					:	Dict     )       ->				Union[str, Any]:



           '''simple docstring'''





           __magic_name__ 		=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
           with tempfile.TemporaryDirectory() as tmpdirname:
                 bert_tokenizer.save(os.path.join(_lowerCAmelCase  ,			'tokenizer.json'     )     )
                 PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowerCAmelCase  ,			'tokenizer.json'     )     )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase :				str						    =       {
    '''configuration_bart''': ['''BART_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BartConfig''', '''BartOnnxConfig'''],
    '''tokenization_bart''': ['''BartTokenizer'''],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Optional[int]						    =       ['''BartTokenizerFast''']

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Any						    =       [
					    '''BART_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''BartForCausalLM''',
					    '''BartForConditionalGeneration''',
					    '''BartForQuestionAnswering''',
					    '''BartForSequenceClassification''',
					    '''BartModel''',
					    '''BartPretrainedModel''',
					    '''PretrainedBartModel''',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Tuple						    =       [
					    '''TFBartForConditionalGeneration''',
					    '''TFBartForSequenceClassification''',
					    '''TFBartModel''',
					    '''TFBartPretrainedModel''',
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				List[Any]						    =       [
					    '''FlaxBartDecoderPreTrainedModel''',
					    '''FlaxBartForCausalLM''',
					    '''FlaxBartForConditionalGeneration''',
					    '''FlaxBartForQuestionAnswering''',
					    '''FlaxBartForSequenceClassification''',
					    '''FlaxBartModel''',
					    '''FlaxBartPreTrainedModel''',
					]

if TYPE_CHECKING:
					from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig, BartOnnxConfig
					from .tokenization_bart import BartTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_bart_fast import BartTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_bart import (
										    BART_PRETRAINED_MODEL_ARCHIVE_LIST,
										    BartForCausalLM,
										    BartForConditionalGeneration,
										    BartForQuestionAnswering,
										    BartForSequenceClassification,
										    BartModel,
										    BartPretrainedModel,
										    PretrainedBartModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_bart import (
										    TFBartForConditionalGeneration,
										    TFBartForSequenceClassification,
										    TFBartModel,
										    TFBartPretrainedModel,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_bart import (
										    FlaxBartDecoderPreTrainedModel,
										    FlaxBartForCausalLM,
										    FlaxBartForConditionalGeneration,
										    FlaxBartForQuestionAnswering,
										    FlaxBartForSequenceClassification,
										    FlaxBartModel,
										    FlaxBartPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase :				Optional[int]						    =       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

