'''simple docstring'''




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
     from tokenizers import Tokenizer
     from tokenizers.models import WordPiece

class   _lowercase(     unittest.TestCase				):



   """simple docstring"""





   def 			snake_case	(							self:    int			,a:    Dict	):
    __UpperCAmelCase         =					list(tokenizer_class.max_model_input_sizes.keys()	)
    for model_name in sa_models[:1]:
     __UpperCAmelCase         =					tokenizer_class.from_pretrained(a	)
     self.assertIsNotNone(a	)
     self.assertIsInstance(a			,a	)
     self.assertIsInstance(a			,a	)

     for special_tok in tokenizer.all_special_tokens:
      self.assertIsInstance(a			,a	)
      __UpperCAmelCase         =					tokenizer.convert_tokens_to_ids(a	)
      self.assertIsInstance(a			,a	)





   def 			snake_case	(							self:    Any			,a:    BatchEncoding			,a:    Optional[Callable] = None	):
    __UpperCAmelCase         =					pickle.dumps(a	)
    self.assertIsNotNone(a	)

    __UpperCAmelCase         =					pickle.loads(a	)

    # Ensure is_fast is correctly restored
    self.assertEqual(be_restored.is_fast			,be_original.is_fast	)

    # Ensure encodings are potentially correctly restored
    if be_original.is_fast:
     self.assertIsNotNone(be_restored.encodings	)
    else:
     self.assertIsNone(be_restored.encodings	)

    # Ensure the keys are the same
    for original_v, restored_v in zip(be_original.values()			,be_restored.values()	):
     if equal_op:
      self.assertTrue(equal_op(a			,a	)	)
     else:
      self.assertEqual(a			,a	)





   @slow
   def 			snake_case	(							self:    Dict	):
    self.check_tokenizer_from_pretrained(a	)





   def 			snake_case	(							self:    Optional[Any]	):
    self.assertEqual(TensorType('tf'	)			,TensorType.TENSORFLOW	)
    self.assertEqual(TensorType('pt'	)			,TensorType.PYTORCH	)
    self.assertEqual(TensorType('np'	)			,TensorType.NUMPY	)





   @require_tokenizers
   def 			snake_case	(							self:    Optional[Any]	):
    import numpy as np

    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
    __UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

    # Python no tensor
    with self.subTest('BatchEncoding (Python, return_tensors=None)'	):
     self.assert_dump_and_restore(tokenizer_p('Small example to encode'	)	)

    with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'	):
     self.assert_dump_and_restore(
         tokenizer_p('Small example to encode'			,return_tensors=TensorType.NUMPY	)			,np.array_equal	)

    with self.subTest('BatchEncoding (Rust, return_tensors=None)'	):
     self.assert_dump_and_restore(tokenizer_r('Small example to encode'	)	)

    with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'	):
     self.assert_dump_and_restore(
         tokenizer_r('Small example to encode'			,return_tensors=TensorType.NUMPY	)			,np.array_equal	)





   @require_tf
   @require_tokenizers
   def 			snake_case	(							self:    Optional[Any]	):
    import tensorflow as tf

    def tf_array_equals(a:    Union[str, Any]			,a:    Any	):
     return tf.reduce_all(tf.equal(a			,a	)	)

    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
    __UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

    with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'	):
     self.assert_dump_and_restore(
         tokenizer_p('Small example to encode'			,return_tensors=TensorType.TENSORFLOW	)			,a	)

    with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'	):
     self.assert_dump_and_restore(
         tokenizer_r('Small example to encode'			,return_tensors=TensorType.TENSORFLOW	)			,a	)





   @require_torch
   @require_tokenizers
   def 			snake_case	(							self:    Optional[Any]	):
    import torch

    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
    __UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

    with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'	):
     self.assert_dump_and_restore(
         tokenizer_p('Small example to encode'			,return_tensors=TensorType.PYTORCH	)			,torch.equal	)

    with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'	):
     self.assert_dump_and_restore(
         tokenizer_r('Small example to encode'			,return_tensors=TensorType.PYTORCH	)			,torch.equal	)





   @require_tokenizers
   def 			snake_case	(							self:    Tuple	):
    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
    __UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

    with self.subTest('Python Tokenizer'	):
     self.assertFalse(tokenizer_p('Small example to_encode'	).is_fast	)

    with self.subTest('Rust Tokenizer'	):
     self.assertTrue(tokenizer_r('Small example to_encode'	).is_fast	)





   @require_tokenizers
   def 			snake_case	(							self:    Optional[int]	):
    __UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)
    __UpperCAmelCase         =					tokenizer_r(['Test', '\xad', 'test']			,is_split_into_words=a	)

    self.assertEqual(encoded.word_to_tokens(0	)			,TokenSpan(start=1			,end=2	)	)
    self.assertEqual(encoded.word_to_tokens(1	)			,a	)
    self.assertEqual(encoded.word_to_tokens(2	)			,TokenSpan(start=2			,end=3	)	)





   def 			snake_case	(							self:    Optional[Any]	):
    __UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'	)
    self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
    # test converting the converted
    with CaptureStderr() as cs:
     __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'	)
    self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

    __UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'			,prepend_batch_axis=a	)
    self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





   @require_torch
   def 			snake_case	(							self:    Optional[Any]	):
    __UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'	)
    self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
    # test converting the converted
    with CaptureStderr() as cs:
     __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'	)
    self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

    __UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'			,prepend_batch_axis=a	)
    self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





   @require_tf
   def 			snake_case	(							self:    List[Any]	):
    __UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'	)
    self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
    # test converting the converted
    with CaptureStderr() as cs:
     __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'	)
    self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

    __UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'			,prepend_batch_axis=a	)
    self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





   @require_flax
   def 			snake_case	(							self:    Any	):
    __UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'	)
    self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
    # test converting the converted
    with CaptureStderr() as cs:
     __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'	)
    self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

    __UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
    __UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'			,prepend_batch_axis=a	)
    self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
    self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





   def 			snake_case	(							self:    List[Any]	):
    __UpperCAmelCase         =					[{'input_ids': np.array([0, 1, 2]	)}, {'input_ids': np.array([0, 1, 2, 3]	)}]
    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
    self.assertTrue(isinstance(batch['input_ids']			,np.ndarray	)	)
    self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='np'	)
    self.assertTrue(isinstance(batch['input_ids']			,np.ndarray	)	)
    self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





   @require_torch
   def 			snake_case	(							self:    Optional[Any]	):
    import torch

    __UpperCAmelCase         =					[{'input_ids': torch.tensor([0, 1, 2]	)}, {'input_ids': torch.tensor([0, 1, 2, 3]	)}]
    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
    self.assertTrue(isinstance(batch['input_ids']			,torch.Tensor	)	)
    self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='pt'	)
    self.assertTrue(isinstance(batch['input_ids']			,torch.Tensor	)	)
    self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





   @require_tf
   def 			snake_case	(							self:    Union[str, Any]	):
    import tensorflow as tf

    __UpperCAmelCase         =					[{'input_ids': tf.constant([0, 1, 2]	)}, {'input_ids': tf.constant([0, 1, 2, 3]	)}]
    __UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
    self.assertTrue(isinstance(batch['input_ids']			,tf.Tensor	)	)
    self.assertEqual(batch['input_ids'].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
    __UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='tf'	)
    self.assertTrue(isinstance(batch['input_ids']			,tf.Tensor	)	)
    self.assertEqual(batch['input_ids'].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





   @require_tokenizers
   def 			snake_case	(							self:    List[str]	):
    __UpperCAmelCase         =					Tokenizer(WordPiece(unk_token='[UNK]'	)	)
    PreTrainedTokenizerFast(tokenizer_object=a	)





   @require_tokenizers
   def 			snake_case	(							self:    str	):
    __UpperCAmelCase         =					Tokenizer(WordPiece(unk_token='[UNK]'	)	)
    with tempfile.TemporaryDirectory() as tmpdirname:
     bert_tokenizer.save(os.path.join(a			,'tokenizer.json'	)	)
     PreTrainedTokenizerFast(tokenizer_file=os.path.join(a			,'tokenizer.json'	)	)




'''simple docstring'''




def 						__snake_case (       lowerCAmelCase       :     str			):
 __UpperCAmelCase         =					5381
 for x in s:
  __UpperCAmelCase         =					((hash_value << 5) + hash_value) + ord(lowerCAmelCase			)
 return hash_value & 0Xffff_ffff



