





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
  from tokenizers import Tokenizer
  from tokenizers.models import WordPiece

class 		__a							(	unittest.TestCase      ):



       def  A							(     self	:   Dict					,			UpperCAmelCase	:   Any			):
           lowerCAmelCase_					:	Optional[int]						=   list(tokenizer_class.max_model_input_sizes.keys()			)
           for model_name in sa_models[:1]:
               lowerCAmelCase_					:	int						=   tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_			)
               self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)
               self.assertIsInstance(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)
               self.assertIsInstance(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)

               for special_tok in tokenizer.all_special_tokens:
                   self.assertIsInstance(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)
                   lowerCAmelCase_					:	List[str]						=   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
                   self.assertIsInstance(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)



       def  A							(     self	:   Optional[Any]					,			UpperCAmelCase	:   BatchEncoding					,			UpperCAmelCase	:   Optional[Callable] = None			):
           lowerCAmelCase_					:	Dict						=   pickle.dumps(SCREAMING_SNAKE_CASE_			)
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)

           lowerCAmelCase_					:	str						=   pickle.loads(SCREAMING_SNAKE_CASE_			)

           # Ensure is_fast is correctly restored
           self.assertEqual(be_restored.is_fast					,			be_original.is_fast			)

           # Ensure encodings are potentially correctly restored
           if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings			)
           else:
               self.assertIsNone(be_restored.encodings			)

           # Ensure the keys are the same
           for original_v, restored_v in zip(be_original.values()					,			be_restored.values()			):
               if equal_op:
                   self.assertTrue(equal_op(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)			)
               else:
                   self.assertEqual(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)



       @slow
       def  A							(     self	:   Union[str, Any]			):
           self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE_			)



       def  A							(     self	:   str			):
           self.assertEqual(TensorType("""tf"""			)					,			TensorType.TENSORFLOW			)
           self.assertEqual(TensorType("""pt"""			)					,			TensorType.PYTORCH			)
           self.assertEqual(TensorType("""np"""			)					,			TensorType.NUMPY			)



       @require_tokenizers
       def  A							(     self	:   int			):
           import numpy as np

           lowerCAmelCase_					:	List[Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)
           lowerCAmelCase_					:	Tuple						=   BertTokenizerFast.from_pretrained("""bert-base-cased"""			)

           # Python no tensor
           with self.subTest("""BatchEncoding (Python, return_tensors=None)"""			):
               self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""			)			)

           with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""			):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,			return_tensors=TensorType.NUMPY			)					,			np.array_equal			)

           with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""			):
               self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""			)			)

           with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""			):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,			return_tensors=TensorType.NUMPY			)					,			np.array_equal			)



       @require_tf
       @require_tokenizers
       def  A							(     self	:   Optional[Any]			):
           import tensorflow as tf

           def tf_array_equals(UpperCAmelCase	:   Any					,			UpperCAmelCase	:   str			):
               return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE_					,			SCREAMING_SNAKE_CASE_			)			)

           lowerCAmelCase_					:	List[Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)
           lowerCAmelCase_					:	Tuple						=   BertTokenizerFast.from_pretrained("""bert-base-cased"""			)

           with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""			):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,			return_tensors=TensorType.TENSORFLOW			)					,			SCREAMING_SNAKE_CASE_			)

           with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""			):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,			return_tensors=TensorType.TENSORFLOW			)					,			SCREAMING_SNAKE_CASE_			)



       @require_torch
       @require_tokenizers
       def  A							(     self	:   List[str]			):
           import torch

           lowerCAmelCase_					:	Union[str, Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)
           lowerCAmelCase_					:	List[Any]						=   BertTokenizerFast.from_pretrained("""bert-base-cased"""			)

           with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""			):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,			return_tensors=TensorType.PYTORCH			)					,			torch.equal			)

           with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""			):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,			return_tensors=TensorType.PYTORCH			)					,			torch.equal			)



       @require_tokenizers
       def  A							(     self	:   Any			):
           lowerCAmelCase_					:	List[Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)
           lowerCAmelCase_					:	Dict						=   BertTokenizerFast.from_pretrained("""bert-base-cased"""			)

           with self.subTest("""Python Tokenizer"""			):
               self.assertFalse(tokenizer_p("""Small example to_encode"""			).is_fast			)

           with self.subTest("""Rust Tokenizer"""			):
               self.assertTrue(tokenizer_r("""Small example to_encode"""			).is_fast			)



       @require_tokenizers
       def  A							(     self	:   Any			):
           lowerCAmelCase_					:	str						=   BertTokenizerFast.from_pretrained("""bert-base-cased"""			)
           lowerCAmelCase_					:	Optional[int]						=   tokenizer_r(["""Test""", """\xad""", """test"""]					,			is_split_into_words=SCREAMING_SNAKE_CASE_			)

           self.assertEqual(encoded.word_to_tokens(0			)					,			TokenSpan(start=1					,			end=2			)			)
           self.assertEqual(encoded.word_to_tokens(1			)					,			SCREAMING_SNAKE_CASE_			)
           self.assertEqual(encoded.word_to_tokens(2			)					,			TokenSpan(start=2					,			end=3			)			)



       def  A							(     self	:   str			):
           lowerCAmelCase_					:	Union[str, Any]						=   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}			)
           lowerCAmelCase_					:	Optional[int]						=   batch.convert_to_tensors(tensor_type="""np"""			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(2, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(2,)			)
           # test converting the converted
           with CaptureStderr() as cs:
               lowerCAmelCase_					:	int						=   batch.convert_to_tensors(tensor_type="""np"""			)
           self.assertFalse(len(cs.err			)					,			msg=F'should have no warning, but got {cs.err}'			)

           lowerCAmelCase_					:	Any						=   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}			)
           lowerCAmelCase_					:	Optional[int]						=   batch.convert_to_tensors(tensor_type="""np"""					,			prepend_batch_axis=SCREAMING_SNAKE_CASE_			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(1, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(1,)			)



       @require_torch
       def  A							(     self	:   Dict			):
           lowerCAmelCase_					:	Tuple						=   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}			)
           lowerCAmelCase_					:	Dict						=   batch.convert_to_tensors(tensor_type="""pt"""			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(2, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(2,)			)
           # test converting the converted
           with CaptureStderr() as cs:
               lowerCAmelCase_					:	Dict						=   batch.convert_to_tensors(tensor_type="""pt"""			)
           self.assertFalse(len(cs.err			)					,			msg=F'should have no warning, but got {cs.err}'			)

           lowerCAmelCase_					:	Optional[Any]						=   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}			)
           lowerCAmelCase_					:	int						=   batch.convert_to_tensors(tensor_type="""pt"""					,			prepend_batch_axis=SCREAMING_SNAKE_CASE_			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(1, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(1,)			)



       @require_tf
       def  A							(     self	:   List[str]			):
           lowerCAmelCase_					:	List[Any]						=   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}			)
           lowerCAmelCase_					:	Any						=   batch.convert_to_tensors(tensor_type="""tf"""			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(2, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(2,)			)
           # test converting the converted
           with CaptureStderr() as cs:
               lowerCAmelCase_					:	Optional[Any]						=   batch.convert_to_tensors(tensor_type="""tf"""			)
           self.assertFalse(len(cs.err			)					,			msg=F'should have no warning, but got {cs.err}'			)

           lowerCAmelCase_					:	Any						=   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}			)
           lowerCAmelCase_					:	str						=   batch.convert_to_tensors(tensor_type="""tf"""					,			prepend_batch_axis=SCREAMING_SNAKE_CASE_			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(1, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(1,)			)



       @require_flax
       def  A							(     self	:   Dict			):
           lowerCAmelCase_					:	int						=   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}			)
           lowerCAmelCase_					:	Union[str, Any]						=   batch.convert_to_tensors(tensor_type="""jax"""			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(2, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(2,)			)
           # test converting the converted
           with CaptureStderr() as cs:
               lowerCAmelCase_					:	int						=   batch.convert_to_tensors(tensor_type="""jax"""			)
           self.assertFalse(len(cs.err			)					,			msg=F'should have no warning, but got {cs.err}'			)

           lowerCAmelCase_					:	List[str]						=   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}			)
           lowerCAmelCase_					:	List[Any]						=   batch.convert_to_tensors(tensor_type="""jax"""					,			prepend_batch_axis=SCREAMING_SNAKE_CASE_			)
           self.assertEqual(tensor_batch["""inputs"""].shape					,			(1, 3)			)
           self.assertEqual(tensor_batch["""labels"""].shape					,			(1,)			)



       def  A							(     self	:   Tuple			):
           lowerCAmelCase_					:	Optional[Any]						=   [{"""input_ids""": np.array([0, 1, 2]			)}, {"""input_ids""": np.array([0, 1, 2, 3]			)}]
           lowerCAmelCase_					:	Union[str, Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)

           lowerCAmelCase_					:	str						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			np.ndarray			)			)
           self.assertEqual(batch["""input_ids"""].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
           lowerCAmelCase_					:	Dict						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_					,			return_tensors="""np"""			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			np.ndarray			)			)
           self.assertEqual(batch["""input_ids"""].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)



       @require_torch
       def  A							(     self	:   Union[str, Any]			):
           import torch

           lowerCAmelCase_					:	List[Any]						=   [{"""input_ids""": torch.tensor([0, 1, 2]			)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]			)}]
           lowerCAmelCase_					:	List[Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)

           lowerCAmelCase_					:	List[str]						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			torch.Tensor			)			)
           self.assertEqual(batch["""input_ids"""].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
           lowerCAmelCase_					:	Tuple						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_					,			return_tensors="""pt"""			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			torch.Tensor			)			)
           self.assertEqual(batch["""input_ids"""].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)



       @require_tf
       def  A							(     self	:   Dict			):
           import tensorflow as tf

           lowerCAmelCase_					:	Any						=   [{"""input_ids""": tf.constant([0, 1, 2]			)}, {"""input_ids""": tf.constant([0, 1, 2, 3]			)}]
           lowerCAmelCase_					:	Union[str, Any]						=   BertTokenizer.from_pretrained("""bert-base-cased"""			)

           lowerCAmelCase_					:	Tuple						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			tf.Tensor			)			)
           self.assertEqual(batch["""input_ids"""].numpy().tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
           lowerCAmelCase_					:	str						=   tokenizer.pad(SCREAMING_SNAKE_CASE_					,			padding=SCREAMING_SNAKE_CASE_					,			return_tensors="""tf"""			)
           self.assertTrue(isinstance(batch["""input_ids"""]					,			tf.Tensor			)			)
           self.assertEqual(batch["""input_ids"""].numpy().tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)



       @require_tokenizers
       def  A							(     self	:   int			):
           lowerCAmelCase_					:	List[str]						=   Tokenizer(WordPiece(unk_token="""[UNK]"""			)			)
           PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_			)



       @require_tokenizers
       def  A							(     self	:   Dict			):
           lowerCAmelCase_					:	Optional[Any]						=   Tokenizer(WordPiece(unk_token="""[UNK]"""			)			)
           with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE_					,			"""tokenizer.json"""			)			)
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE_					,			"""tokenizer.json"""			)			)









def 			__UpperCamelCase				(			lowercase__	:       set		,    lowercase__	:       set		)	->      set:
    '''simple docstring'''





    assert isinstance(lowercase__		,    lowercase__		), f'The input value of [set_a={set_a}] is not a set'
    assert isinstance(lowercase__		,    lowercase__		), f'The input value of [set_b={set_b}] is not a set'

    return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
  from doctest import testmod

  testmod()


