
"""simple docstring"""


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
							from tokenizers import Tokenizer
							from tokenizers.models import WordPiece

class 					A_			(  unittest.TestCase     ):



							"""simple docstring"""






							def    UpperCAmelCase__							(						self						:Tuple  ,							lowerCamelCase_						:Union[str, Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  Optional[Any]  =list(tokenizer_class.max_model_input_sizes.keys() )
										for model_name in sa_models[:1]:
													lowerCamelCase__			:  Any  =tokenizer_class.from_pretrained(a_ )
													self.assertIsNotNone(a_ )
													self.assertIsInstance(a_  ,							a_ )
													self.assertIsInstance(a_  ,							a_ )

													for special_tok in tokenizer.all_special_tokens:
																self.assertIsInstance(a_  ,							a_ )
																lowerCamelCase__			:  Dict  =tokenizer.convert_tokens_to_ids(a_ )
																self.assertIsInstance(a_  ,							a_ )





							def    UpperCAmelCase__							(						self						:Union[str, Any]  ,							lowerCamelCase_						:BatchEncoding  ,							lowerCamelCase_						:Optional[Callable] = None ):





										"""simple docstring"""

										lowerCamelCase__			:  int  =pickle.dumps(a_ )
										self.assertIsNotNone(a_ )

										lowerCamelCase__			:  List[Any]  =pickle.loads(a_ )

										# Ensure is_fast is correctly restored
										self.assertEqual(be_restored.is_fast  ,							be_original.is_fast )

										# Ensure encodings are potentially correctly restored
										if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings )
										else:
													self.assertIsNone(be_restored.encodings )

										# Ensure the keys are the same
										for original_v, restored_v in zip(be_original.values()  ,							be_restored.values() ):
													if equal_op:
																self.assertTrue(equal_op(a_  ,							a_ ) )
													else:
																self.assertEqual(a_  ,							a_ )





							@slow
							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										self.check_tokenizer_from_pretrained(a_ )





							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										self.assertEqual(TensorType('tf' )  ,							TensorType.TENSORFLOW )
										self.assertEqual(TensorType('pt' )  ,							TensorType.PYTORCH )
										self.assertEqual(TensorType('np' )  ,							TensorType.NUMPY )





							@require_tokenizers
							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										import numpy as np

										lowerCamelCase__			:  Union[str, Any]  =BertTokenizer.from_pretrained('bert-base-cased' )
										lowerCamelCase__			:  List[str]  =BertTokenizerFast.from_pretrained('bert-base-cased' )

										# Python no tensor
										with self.subTest('BatchEncoding (Python, return_tensors=None)' ):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode' ) )

										with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)' ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'  ,							return_tensors=TensorType.NUMPY )  ,							np.array_equal )

										with self.subTest('BatchEncoding (Rust, return_tensors=None)' ):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode' ) )

										with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)' ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'  ,							return_tensors=TensorType.NUMPY )  ,							np.array_equal )





							@require_tf
							@require_tokenizers
							def    UpperCAmelCase__							(						self						:str ):





										"""simple docstring"""

										import tensorflow as tf

										def tf_array_equals(lowerCamelCase_						:Tuple  ,							lowerCamelCase_						:str ):
													return tf.reduce_all(tf.equal(a_  ,							a_ ) )

										lowerCamelCase__			:  Union[str, Any]  =BertTokenizer.from_pretrained('bert-base-cased' )
										lowerCamelCase__			:  Any  =BertTokenizerFast.from_pretrained('bert-base-cased' )

										with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)' ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'  ,							return_tensors=TensorType.TENSORFLOW )  ,							a_ )

										with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)' ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'  ,							return_tensors=TensorType.TENSORFLOW )  ,							a_ )





							@require_torch
							@require_tokenizers
							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										import torch

										lowerCamelCase__			:  Union[str, Any]  =BertTokenizer.from_pretrained('bert-base-cased' )
										lowerCamelCase__			:  Any  =BertTokenizerFast.from_pretrained('bert-base-cased' )

										with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)' ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'  ,							return_tensors=TensorType.PYTORCH )  ,							torch.equal )

										with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)' ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'  ,							return_tensors=TensorType.PYTORCH )  ,							torch.equal )





							@require_tokenizers
							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  List[Any]  =BertTokenizer.from_pretrained('bert-base-cased' )
										lowerCamelCase__			:  Tuple  =BertTokenizerFast.from_pretrained('bert-base-cased' )

										with self.subTest('Python Tokenizer' ):
													self.assertFalse(tokenizer_p('Small example to_encode' ).is_fast )

										with self.subTest('Rust Tokenizer' ):
													self.assertTrue(tokenizer_r('Small example to_encode' ).is_fast )





							@require_tokenizers
							def    UpperCAmelCase__							(						self						:str ):





										"""simple docstring"""

										lowerCamelCase__			:  Optional[int]  =BertTokenizerFast.from_pretrained('bert-base-cased' )
										lowerCamelCase__			:  Optional[int]  =tokenizer_r(['Test', '\xad', 'test']  ,							is_split_into_words=a_ )

										self.assertEqual(encoded.word_to_tokens(0 )  ,							TokenSpan(start=1  ,							end=2 ) )
										self.assertEqual(encoded.word_to_tokens(1 )  ,							a_ )
										self.assertEqual(encoded.word_to_tokens(2 )  ,							TokenSpan(start=2  ,							end=3 ) )





							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  Any  =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
										lowerCamelCase__			:  List[str]  =batch.convert_to_tensors(tensor_type='np' )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(2, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(2,) )
										# test converting the converted
										with CaptureStderr() as cs:
													lowerCamelCase__			:  Union[str, Any]  =batch.convert_to_tensors(tensor_type='np' )
										self.assertFalse(len(cs.err )  ,							msg=f"""should have no warning, but got {cs.err}""" )

										lowerCamelCase__			:  Optional[int]  =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
										lowerCamelCase__			:  str  =batch.convert_to_tensors(tensor_type='np'  ,							prepend_batch_axis=a_ )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(1, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(1,) )





							@require_torch
							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  str  =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
										lowerCamelCase__			:  Union[str, Any]  =batch.convert_to_tensors(tensor_type='pt' )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(2, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(2,) )
										# test converting the converted
										with CaptureStderr() as cs:
													lowerCamelCase__			:  int  =batch.convert_to_tensors(tensor_type='pt' )
										self.assertFalse(len(cs.err )  ,							msg=f"""should have no warning, but got {cs.err}""" )

										lowerCamelCase__			:  List[Any]  =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
										lowerCamelCase__			:  List[Any]  =batch.convert_to_tensors(tensor_type='pt'  ,							prepend_batch_axis=a_ )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(1, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(1,) )





							@require_tf
							def    UpperCAmelCase__							(						self						:Optional[Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  Any  =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
										lowerCamelCase__			:  Optional[int]  =batch.convert_to_tensors(tensor_type='tf' )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(2, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(2,) )
										# test converting the converted
										with CaptureStderr() as cs:
													lowerCamelCase__			:  Any  =batch.convert_to_tensors(tensor_type='tf' )
										self.assertFalse(len(cs.err )  ,							msg=f"""should have no warning, but got {cs.err}""" )

										lowerCamelCase__			:  List[str]  =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
										lowerCamelCase__			:  Union[str, Any]  =batch.convert_to_tensors(tensor_type='tf'  ,							prepend_batch_axis=a_ )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(1, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(1,) )





							@require_flax
							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  List[Any]  =BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
										lowerCamelCase__			:  str  =batch.convert_to_tensors(tensor_type='jax' )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(2, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(2,) )
										# test converting the converted
										with CaptureStderr() as cs:
													lowerCamelCase__			:  List[Any]  =batch.convert_to_tensors(tensor_type='jax' )
										self.assertFalse(len(cs.err )  ,							msg=f"""should have no warning, but got {cs.err}""" )

										lowerCamelCase__			:  List[str]  =BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
										lowerCamelCase__			:  Union[str, Any]  =batch.convert_to_tensors(tensor_type='jax'  ,							prepend_batch_axis=a_ )
										self.assertEqual(tensor_batch['inputs'].shape  ,							(1, 3) )
										self.assertEqual(tensor_batch['labels'].shape  ,							(1,) )





							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										lowerCamelCase__			:  Tuple  =[{'input_ids': np.array([0, 1, 2] )}, {'input_ids': np.array([0, 1, 2, 3] )}]
										lowerCamelCase__			:  Dict  =BertTokenizer.from_pretrained('bert-base-cased' )

										lowerCamelCase__			:  Optional[Any]  =tokenizer.pad(a_  ,							padding=a_ )
										self.assertTrue(isinstance(batch['input_ids']  ,							np.ndarray ) )
										self.assertEqual(batch['input_ids'].tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
										lowerCamelCase__			:  Any  =tokenizer.pad(a_  ,							padding=a_  ,							return_tensors='np' )
										self.assertTrue(isinstance(batch['input_ids']  ,							np.ndarray ) )
										self.assertEqual(batch['input_ids'].tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





							@require_torch
							def    UpperCAmelCase__							(						self						:List[Any] ):





										"""simple docstring"""

										import torch

										lowerCamelCase__			:  List[str]  =[{'input_ids': torch.tensor([0, 1, 2] )}, {'input_ids': torch.tensor([0, 1, 2, 3] )}]
										lowerCamelCase__			:  Tuple  =BertTokenizer.from_pretrained('bert-base-cased' )

										lowerCamelCase__			:  Dict  =tokenizer.pad(a_  ,							padding=a_ )
										self.assertTrue(isinstance(batch['input_ids']  ,							torch.Tensor ) )
										self.assertEqual(batch['input_ids'].tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
										lowerCamelCase__			:  int  =tokenizer.pad(a_  ,							padding=a_  ,							return_tensors='pt' )
										self.assertTrue(isinstance(batch['input_ids']  ,							torch.Tensor ) )
										self.assertEqual(batch['input_ids'].tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





							@require_tf
							def    UpperCAmelCase__							(						self						:int ):





										"""simple docstring"""

										import tensorflow as tf

										lowerCamelCase__			:  Optional[Any]  =[{'input_ids': tf.constant([0, 1, 2] )}, {'input_ids': tf.constant([0, 1, 2, 3] )}]
										lowerCamelCase__			:  List[Any]  =BertTokenizer.from_pretrained('bert-base-cased' )

										lowerCamelCase__			:  Optional[Any]  =tokenizer.pad(a_  ,							padding=a_ )
										self.assertTrue(isinstance(batch['input_ids']  ,							tf.Tensor ) )
										self.assertEqual(batch['input_ids'].numpy().tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
										lowerCamelCase__			:  Any  =tokenizer.pad(a_  ,							padding=a_  ,							return_tensors='tf' )
										self.assertTrue(isinstance(batch['input_ids']  ,							tf.Tensor ) )
										self.assertEqual(batch['input_ids'].numpy().tolist()  ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





							@require_tokenizers
							def    UpperCAmelCase__							(						self						:List[str] ):





										"""simple docstring"""

										lowerCamelCase__			:  Dict  =Tokenizer(WordPiece(unk_token='[UNK]' ) )
										PreTrainedTokenizerFast(tokenizer_object=a_ )





							@require_tokenizers
							def    UpperCAmelCase__							(						self						:Optional[int] ):





										"""simple docstring"""

										lowerCamelCase__			:  Optional[int]  =Tokenizer(WordPiece(unk_token='[UNK]' ) )
										with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(a_  ,							'tokenizer.json' ) )
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_  ,							'tokenizer.json' ) )

"""simple docstring"""


import unittest
import warnings

from datasets.utils import experimental






@experimental
def      lowerCAmelCase_     (	)     ->Optional[Any]:
			return "success"

class 					A_			(  unittest.TestCase     ):



							"""simple docstring"""






							def    UpperCAmelCase__							(						self						:int ):





										"""simple docstring"""

										with warnings.catch_warnings(record=lowerCamelCase_ ) as w:
													warnings.simplefilter('always' )
													self.assertEqual(dummy_function()  ,							'success' )
										self.assertEqual(len(lowerCamelCase_ )  ,							1 )