
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece

class      lowerCAmelCase__  (			unittest.TestCase    ):






		def 				__A (	self	:							Optional[int]							,    SCREAMING_SNAKE_CASE__	:							str						)  ->							int:
						__lowerCamelCase					      =			list(tokenizer_class.max_model_input_sizes.keys()						)
						for model_name in sa_models[:1]:
										__lowerCamelCase					      =			tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__						)
										self.assertIsNotNone(SCREAMING_SNAKE_CASE__						)
										self.assertIsInstance(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)
										self.assertIsInstance(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)

										for special_tok in tokenizer.all_special_tokens:
														self.assertIsInstance(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)
														__lowerCamelCase					      =			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__						)
														self.assertIsInstance(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)






		def 				__A (	self	:							Union[str, Any]							,    SCREAMING_SNAKE_CASE__	:							BatchEncoding							,    SCREAMING_SNAKE_CASE__	:							Optional[Callable] = None						)  ->							str:
						__lowerCamelCase					      =			pickle.dumps(SCREAMING_SNAKE_CASE__						)
						self.assertIsNotNone(SCREAMING_SNAKE_CASE__						)

						__lowerCamelCase					      =			pickle.loads(SCREAMING_SNAKE_CASE__						)

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast							,    be_original.is_fast						)

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
										self.assertIsNotNone(be_restored.encodings						)
						else:
										self.assertIsNone(be_restored.encodings						)

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()							,    be_restored.values()						):
										if equal_op:
														self.assertTrue(equal_op(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)						)
										else:
														self.assertEqual(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)






		@slow
		def 				__A (	self	:							List[str]						)  ->							Tuple:
						self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE__						)






		def 				__A (	self	:							str						)  ->							str:
						self.assertEqual(TensorType('''tf'''						)							,    TensorType.TENSORFLOW						)
						self.assertEqual(TensorType('''pt'''						)							,    TensorType.PYTORCH						)
						self.assertEqual(TensorType('''np'''						)							,    TensorType.NUMPY						)






		@require_tokenizers
		def 				__A (	self	:							Dict						)  ->							Union[str, Any]:
						import numpy as np

						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)
						__lowerCamelCase					      =			BertTokenizerFast.from_pretrained('''bert-base-cased'''						)

						# Python no tensor
						with self.subTest('''BatchEncoding (Python, return_tensors=None)'''						):
										self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''						)						)

						with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''						):
										self.assert_dump_and_restore(
										    tokenizer_p('''Small example to encode'''							,    return_tensors=TensorType.NUMPY						)							,    np.array_equal						)

						with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''						):
										self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''						)						)

						with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''						):
										self.assert_dump_and_restore(
										    tokenizer_r('''Small example to encode'''							,    return_tensors=TensorType.NUMPY						)							,    np.array_equal						)






		@require_tf
		@require_tokenizers
		def 				__A (	self	:							str						)  ->							str:
						import tensorflow as tf

						def tf_array_equals(SCREAMING_SNAKE_CASE__	:							List[str]							,    SCREAMING_SNAKE_CASE__	:							Optional[int]						):
										return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE__							,    SCREAMING_SNAKE_CASE__						)						)

						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)
						__lowerCamelCase					      =			BertTokenizerFast.from_pretrained('''bert-base-cased'''						)

						with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''						):
										self.assert_dump_and_restore(
										    tokenizer_p('''Small example to encode'''							,    return_tensors=TensorType.TENSORFLOW						)							,    SCREAMING_SNAKE_CASE__						)

						with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''						):
										self.assert_dump_and_restore(
										    tokenizer_r('''Small example to encode'''							,    return_tensors=TensorType.TENSORFLOW						)							,    SCREAMING_SNAKE_CASE__						)






		@require_torch
		@require_tokenizers
		def 				__A (	self	:							List[str]						)  ->							Optional[Any]:
						import torch

						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)
						__lowerCamelCase					      =			BertTokenizerFast.from_pretrained('''bert-base-cased'''						)

						with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''						):
										self.assert_dump_and_restore(
										    tokenizer_p('''Small example to encode'''							,    return_tensors=TensorType.PYTORCH						)							,    torch.equal						)

						with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''						):
										self.assert_dump_and_restore(
										    tokenizer_r('''Small example to encode'''							,    return_tensors=TensorType.PYTORCH						)							,    torch.equal						)






		@require_tokenizers
		def 				__A (	self	:							int						)  ->							str:
						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)
						__lowerCamelCase					      =			BertTokenizerFast.from_pretrained('''bert-base-cased'''						)

						with self.subTest('''Python Tokenizer'''						):
										self.assertFalse(tokenizer_p('''Small example to_encode'''						).is_fast						)

						with self.subTest('''Rust Tokenizer'''						):
										self.assertTrue(tokenizer_r('''Small example to_encode'''						).is_fast						)






		@require_tokenizers
		def 				__A (	self	:							int						)  ->							Optional[Any]:
						__lowerCamelCase					      =			BertTokenizerFast.from_pretrained('''bert-base-cased'''						)
						__lowerCamelCase					      =			tokenizer_r(['''Test''', '''\xad''', '''test''']							,    is_split_into_words=SCREAMING_SNAKE_CASE__						)

						self.assertEqual(encoded.word_to_tokens(0						)							,    TokenSpan(start=1							,    end=2						)						)
						self.assertEqual(encoded.word_to_tokens(1						)							,    SCREAMING_SNAKE_CASE__						)
						self.assertEqual(encoded.word_to_tokens(2						)							,    TokenSpan(start=2							,    end=3						)						)






		def 				__A (	self	:							Dict						)  ->							Optional[Any]:
						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''np'''						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (2, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (2,)						)
						# test converting the converted
						with CaptureStderr() as cs:
										__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''np'''						)
						self.assertFalse(len(cs.err						)							,    msg=f'''should have no warning, but got {cs.err}'''						)

						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''np'''							,    prepend_batch_axis=SCREAMING_SNAKE_CASE__						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (1, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (1,)						)






		@require_torch
		def 				__A (	self	:							Optional[int]						)  ->							Optional[int]:
						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''pt'''						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (2, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (2,)						)
						# test converting the converted
						with CaptureStderr() as cs:
										__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''pt'''						)
						self.assertFalse(len(cs.err						)							,    msg=f'''should have no warning, but got {cs.err}'''						)

						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''pt'''							,    prepend_batch_axis=SCREAMING_SNAKE_CASE__						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (1, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (1,)						)






		@require_tf
		def 				__A (	self	:							Optional[int]						)  ->							Optional[Any]:
						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''tf'''						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (2, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (2,)						)
						# test converting the converted
						with CaptureStderr() as cs:
										__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''tf'''						)
						self.assertFalse(len(cs.err						)							,    msg=f'''should have no warning, but got {cs.err}'''						)

						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''tf'''							,    prepend_batch_axis=SCREAMING_SNAKE_CASE__						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (1, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (1,)						)






		@require_flax
		def 				__A (	self	:							Optional[Any]						)  ->							List[str]:
						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''jax'''						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (2, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (2,)						)
						# test converting the converted
						with CaptureStderr() as cs:
										__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''jax'''						)
						self.assertFalse(len(cs.err						)							,    msg=f'''should have no warning, but got {cs.err}'''						)

						__lowerCamelCase					      =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}						)
						__lowerCamelCase					      =			batch.convert_to_tensors(tensor_type='''jax'''							,    prepend_batch_axis=SCREAMING_SNAKE_CASE__						)
						self.assertEqual(tensor_batch['''inputs'''].shape							,    (1, 3)						)
						self.assertEqual(tensor_batch['''labels'''].shape							,    (1,)						)






		def 				__A (	self	:							List[Any]						)  ->							Optional[Any]:
						__lowerCamelCase					      =			[{'''input_ids''': np.array([0, 1, 2]						)}, {'''input_ids''': np.array([0, 1, 2, 3]						)}]
						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)

						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    np.ndarray						)						)
						self.assertEqual(batch['''input_ids'''].tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__							,    return_tensors='''np'''						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    np.ndarray						)						)
						self.assertEqual(batch['''input_ids'''].tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






		@require_torch
		def 				__A (	self	:							Optional[Any]						)  ->							str:
						import torch

						__lowerCamelCase					      =			[{'''input_ids''': torch.tensor([0, 1, 2]						)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]						)}]
						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)

						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    torch.Tensor						)						)
						self.assertEqual(batch['''input_ids'''].tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__							,    return_tensors='''pt'''						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    torch.Tensor						)						)
						self.assertEqual(batch['''input_ids'''].tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






		@require_tf
		def 				__A (	self	:							List[str]						)  ->							Optional[int]:
						import tensorflow as tf

						__lowerCamelCase					      =			[{'''input_ids''': tf.constant([0, 1, 2]						)}, {'''input_ids''': tf.constant([0, 1, 2, 3]						)}]
						__lowerCamelCase					      =			BertTokenizer.from_pretrained('''bert-base-cased'''						)

						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    tf.Tensor						)						)
						self.assertEqual(batch['''input_ids'''].numpy().tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
						__lowerCamelCase					      =			tokenizer.pad(SCREAMING_SNAKE_CASE__							,    padding=SCREAMING_SNAKE_CASE__							,    return_tensors='''tf'''						)
						self.assertTrue(isinstance(batch['''input_ids''']							,    tf.Tensor						)						)
						self.assertEqual(batch['''input_ids'''].numpy().tolist()							,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






		@require_tokenizers
		def 				__A (	self	:							Union[str, Any]						)  ->							Tuple:
						__lowerCamelCase					      =			Tokenizer(WordPiece(unk_token='''[UNK]'''						)						)
						PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE__						)







		@require_tokenizers
		def 				__A (	self	:							Optional[Any]						)  ->							Dict:
						__lowerCamelCase					      =			Tokenizer(WordPiece(unk_token='''[UNK]'''						)						)
						with tempfile.TemporaryDirectory() as tmpdirname:
										bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE__							,    '''tokenizer.json'''						)						)
										PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE__							,    '''tokenizer.json'''						)						)





from ..utils import requires_backends


SCREAMING_SNAKE_CASE__     :    Union[str, Any]      							=					None

class      lowerCAmelCase__  :






		def __init__(	self	:							int							,    *SCREAMING_SNAKE_CASE__	:							Dict							,    **SCREAMING_SNAKE_CASE__	:							Tuple						)  ->							Optional[Any]:
						requires_backends(self							,    ['''detectron2''']						)







		@classmethod
		def 				__A (	cls	:							Dict							,    *SCREAMING_SNAKE_CASE__	:							Dict							,    **SCREAMING_SNAKE_CASE__	:							Tuple						)  ->							Tuple:
						requires_backends(cls							,    ['''detectron2''']						)



