




'''simple docstring'''






from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
      import numpy as np
      import tensorflow as tf

      from transformers import (
          TF_MODEL_FOR_CAUSAL_LM_MAPPING,
          TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
          TF_MODEL_FOR_MASKED_LM_MAPPING,
          TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
          TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
          TF_MODEL_FOR_PRETRAINING_MAPPING,
          TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
          TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
          TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
          TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
          TFSharedEmbeddings,
      )

      if _tf_gpu_memory_limit is not None:
            lowercase	:      Optional[int]				=       tf.config.list_physical_devices('''GPU''')
            for gpu in gpus:
                  # Restrict TensorFlow to only allocate x GB of memory on the GPUs
                  try:
                        tf.config.set_logical_device_configuration(
                            gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
                        )
                        lowercase	:      List[str]				=       tf.config.list_logical_devices('''GPU''')
                        print('''Logical GPUs''', logical_gpus)
                  except RuntimeError as e:
                        # Virtual devices must be set before GPUs have been initialized
                        print(e)

@require_tf
class   SCREAMING_SNAKE_CASE_		:
 __snake_case	:     int			   =      None
 __snake_case	:     List[str]			   =      ()
 __snake_case	:     Optional[int]			   =      ()
 __snake_case	:     Tuple			   =      True
 __snake_case	:     Optional[int]			   =      True
 __snake_case	:     Any			   =      True
 __snake_case	:     Union[str, Any]			   =      False



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A=False ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      copy.deepcopy(A )

        if model_class in get_values(A ):
               lowercase  :		Optional[Any]						=      {
                   k: tf.tile(tf.expand_dims(A   ,		1 )   ,		(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1) )
                   if isinstance(A   ,		tf.Tensor ) and v.ndim > 0
                   else v
                   for k, v in inputs_dict.items()
               }

        if return_labels:
               if model_class in get_values(A ):
                      lowercase  :		List[str]						=      tf.ones(self.model_tester.batch_size   ,		dtype=tf.intaa )
               elif model_class in get_values(A ):
                      lowercase  :		Optional[Any]						=      tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa )
                      lowercase  :		Optional[Any]						=      tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa )
               elif model_class in [
                   *get_values(A ),
                   *get_values(A ),
               ]:
                      lowercase  :		Optional[int]						=      tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa )
               elif model_class in get_values(A ):
                      lowercase  :		Optional[int]						=      tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa )
               elif model_class in [
                   *get_values(A ),
                   *get_values(A ),
                   *get_values(A ),
                   *get_values(A ),
                   *get_values(A ),
               ]:
                      lowercase  :		List[str]						=      tf.zeros(
                          (self.model_tester.batch_size, self.model_tester.seq_length)   ,		dtype=tf.intaa )
        return inputs_dict



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		int						=      self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
               lowercase  :		List[str]						=      self._prepare_for_class(A   ,		A )
               lowercase  :		str						=      model_class(A )

               @tf.function
               def run_in_graph_mode():
                      return model(A )

               lowercase  :		Tuple						=      run_in_graph_mode()
               self.assertIsNotNone(A )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		List[str]						=      self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
               lowercase  :		Tuple						=      self._prepare_for_class(A   ,		A )
               lowercase  :		str						=      model_class(A )

               @tf.function(experimental_compile=A )
               def run_in_graph_mode():
                      return model(A )

               lowercase  :		Optional[int]						=      run_in_graph_mode()
               self.assertIsNotNone(A )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Dict						=      self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
               lowercase  :		List[str]						=      model_class(A )
               if getattr(A   ,		'''hf_compute_loss'''   ,		A ):
                      # Test that model correctly compute the loss with kwargs
                      lowercase  :		int						=      self._prepare_for_class(inputs_dict.copy()   ,		A   ,		return_labels=A )
                      # Is there a better way to remove these decoder inputs?
                      lowercase  :		Any						=      {
                          key: val
                          for key, val in prepared_for_class.items()
                          if key not in ('''head_mask''', '''decoder_head_mask''', '''cross_attn_head_mask''', '''decoder_input_ids''')
                      }

                      lowercase  :		str						=      {
                          '''labels''',
                          '''label''',
                          '''label_ids''',
                          '''start_positions''',
                          '''start_position''',
                          '''end_positions''',
                          '''end_position''',
                          '''next_sentence_label''',
                      }
                      lowercase  :		str						=      possible_label_cols.intersection(set(A ) )
                      self.assertGreater(len(A )   ,		0   ,		msg='''No matching label names found!''' )
                      lowercase  :		Optional[Any]						=      {key: val for key, val in prepared_for_class.items() if key in label_names}
                      lowercase  :		Optional[Any]						=      {key: val for key, val in prepared_for_class.items() if key not in label_names}
                      self.assertGreater(len(A )   ,		0 )

                      # Make sure it works with XLA!
                      model.compile(optimizer=tf.keras.optimizers.SGD(0.0 )   ,		jit_compile=A )
                      # Make sure the model fits without crashing regardless of where we pass the labels
                      lowercase  :		Union[str, Any]						=      model.fit(
                          A   ,		validation_data=A   ,		steps_per_epoch=1   ,		validation_steps=1   ,		shuffle=A   ,		verbose=0   ,		)
                      lowercase  :		Any						=      history.history['''loss'''][0]
                      self.assertTrue(not isnan(A ) )
                      lowercase  :		List[str]						=      history.history['''val_loss'''][0]
                      self.assertTrue(not isnan(A ) )

                      # Now test it with separate labels, to make sure that path works in XLA too.
                      lowercase  :		Any						=      model_class(A )
                      model.compile(optimizer=tf.keras.optimizers.SGD(0.0 )   ,		jit_compile=A )
                      lowercase  :		Optional[Any]						=      model.fit(
                          A   ,		A   ,		validation_data=(inputs_minus_labels, labels)   ,		steps_per_epoch=1   ,		validation_steps=1   ,		shuffle=A   ,		verbose=0   ,		)

                      lowercase  :		Optional[Any]						=      history.history['''loss'''][0]
                      self.assertTrue(not isnan(A ) )
                      lowercase  :		Dict						=      history.history['''val_loss'''][0]
                      self.assertTrue(not isnan(A ) )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Optional[Any]						=      self.model_tester.prepare_config_and_inputs_for_common()
        lowercase  :		str						=      True
        lowercase  :		Any						=      True

        if hasattr(A   ,		'''use_cache''' ):
               lowercase  :		List[Any]						=      True

        lowercase  :		Tuple						=      getattr(self.model_tester   ,		'''encoder_seq_length'''   ,		self.model_tester.seq_length )
        lowercase  :		Optional[int]						=      getattr(self.model_tester   ,		'''key_length'''   ,		A )

        for model_class in self.all_model_classes[:2]:
               lowercase  :		Optional[int]						=      self._prepare_for_class(A   ,		A )
               lowercase  :		Optional[Any]						=      model_class(A )
               model.build()
               lowercase  :		List[Any]						=      len(model(A ) )

               for key in list(class_inputs_dict.keys() ):
                      # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
                      if key not in model.input_signature:
                             del class_inputs_dict[key]
                      # Check it's a tensor, in case the inputs dict has some bools in it too
                      elif isinstance(class_inputs_dict[key]   ,		tf.Tensor ) and class_inputs_dict[key].dtype.is_integer:
                             lowercase  :		Union[str, Any]						=      tf.cast(class_inputs_dict[key]   ,		tf.intaa )

               if set(class_inputs_dict.keys() ) != set(model.input_signature.keys() ):
                      continue  # Some models have inputs that the preparation functions don't create, we skip those

               with tempfile.TemporaryDirectory() as tmpdirname:
                      model.save_pretrained(A   ,		saved_model=A )
                      lowercase  :		Optional[int]						=      os.path.join(A   ,		'''saved_model'''   ,		'''1''' )
                      lowercase  :		int						=      tf.keras.models.load_model(A )
                      lowercase  :		List[Any]						=      model(A )

                      if self.is_encoder_decoder:
                             lowercase  :		Union[str, Any]						=      outputs['''encoder_hidden_states''']
                             lowercase  :		Any						=      outputs['''encoder_attentions''']
                      else:
                             lowercase  :		Tuple						=      outputs['''hidden_states''']
                             lowercase  :		Any						=      outputs['''attentions''']

                      self.assertEqual(len(A )   ,		A )

                      lowercase  :		Optional[Any]						=      getattr(
                          self.model_tester   ,		'''expected_num_hidden_layers'''   ,		self.model_tester.num_hidden_layers + 1 )

                      self.assertEqual(len(A )   ,		A )
                      self.assertListEqual(
                          list(output_hidden_states[0].shape[-2:] )   ,		[self.model_tester.seq_length, self.model_tester.hidden_size]   ,		)

                      self.assertEqual(len(A )   ,		self.model_tester.num_hidden_layers )
                      self.assertListEqual(
                          list(output_attentions[0].shape[-3:] )   ,		[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]   ,		)



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        tf.keras.mixed_precision.set_global_policy('''mixed_float16''' )

        # try/finally block to ensure subsequent tests run in float32
        try:
               lowercase,     lowercase  :		Union[str, Any]						=      self.model_tester.prepare_config_and_inputs_for_common()
               for model_class in self.all_model_classes[:2]:
                      lowercase  :		int						=      self._prepare_for_class(A   ,		A )
                      lowercase  :		Optional[int]						=      model_class(A )
                      lowercase  :		Optional[Any]						=      model(A )

                      self.assertIsNotNone(A )
        finally:
               tf.keras.mixed_precision.set_global_policy('''float32''' )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		str						=      self.model_tester.prepare_config_and_inputs_for_common()
        # head_mask and decoder_head_mask has different shapes than other input args
        if "head_mask" in inputs_dict:
               del inputs_dict["head_mask"]
        if "decoder_head_mask" in inputs_dict:
               del inputs_dict["decoder_head_mask"]
        if "cross_attn_head_mask" in inputs_dict:
               del inputs_dict["cross_attn_head_mask"]
        lowercase  :		Any						=      {
            module_member
            for model_class in self.all_model_classes
            for module in (import_module(model_class.__module__ ),)
            for module_member_name in dir(A )
            if module_member_name.endswith('''MainLayer''' )
            for module_member in (getattr(A   ,		A ),)
            if isinstance(A   ,		A )
            and tf.keras.layers.Layer in module_member.__bases__
            and getattr(A   ,		'''_keras_serializable'''   ,		A )
        }

        for main_layer_class in tf_main_layer_classes:
               # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
               if "T5" in main_layer_class.__name__:
                      # Take the same values than in TFT5ModelTester for this shared layer
                      lowercase  :		Union[str, Any]						=      TFSharedEmbeddings(self.model_tester.vocab_size   ,		self.model_tester.hidden_size   ,		name='''shared''' )
                      lowercase  :		Optional[int]						=      False
                      lowercase  :		Optional[Any]						=      main_layer_class(A   ,		embed_tokens=A )
               else:
                      lowercase  :		Optional[Any]						=      main_layer_class(A )

               lowercase  :		Any						=      {
                   name: tf.keras.Input(tensor.shape[1:]   ,		dtype=tensor.dtype ) for name, tensor in inputs_dict.items()
               }

               if hasattr(self.model_tester   ,		'''num_labels''' ):
                      lowercase  :		Dict						=      self.model_tester.num_labels
               else:
                      lowercase  :		Tuple						=      2

               lowercase  :		Dict						=      tf.data.Dataset.from_tensor_slices(
                   (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1) )) ).batch(1 )

               lowercase  :		Union[str, Any]						=      main_layer(A )[0]
               lowercase  :		int						=      tf.keras.layers.Dense(A   ,		activation='''softmax'''   ,		name='''outputs''' )(A )
               lowercase  :		List[Any]						=      tf.keras.models.Model(inputs=A   ,		outputs=[outputs] )

               model.compile(loss='''binary_crossentropy'''   ,		optimizer='''adam'''   ,		metrics=['''binary_accuracy'''] )
               model.fit(A   ,		epochs=1 )

               with tempfile.TemporaryDirectory() as tmpdirname:
                      lowercase  :		int						=      os.path.join(A   ,		'''keras_model.h5''' )
                      model.save(A )
                      if "T5" in main_layer_class.__name__:
                             lowercase  :		str						=      tf.keras.models.load_model(
                                 A   ,		custom_objects={
                                     main_layer_class.__name__: main_layer_class,
                                     '''TFSharedEmbeddings''': TFSharedEmbeddings,
                                 }   ,		)
                      else:
                             lowercase  :		Tuple						=      tf.keras.models.load_model(
                                 A   ,		custom_objects={main_layer_class.__name__: main_layer_class} )
                      assert isinstance(A   ,		tf.keras.Model )
                      model(A )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Any						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes[:2]:
               lowercase  :		Optional[Any]						=      model_class(A )

               lowercase  :		Dict						=      copy.deepcopy(A )

               if not self.is_encoder_decoder:
                      lowercase  :		List[Any]						=      inputs['''input_ids''']
                      del inputs["input_ids"]
               else:
                      lowercase  :		str						=      inputs['''input_ids''']
                      lowercase  :		Optional[int]						=      inputs.get('''decoder_input_ids'''   ,		A )
                      del inputs["input_ids"]
                      inputs.pop('''decoder_input_ids'''   ,		A )

               if not self.is_encoder_decoder:
                      lowercase  :		Union[str, Any]						=      model.get_input_embeddings()(A )
               else:
                      lowercase  :		Any						=      model.get_input_embeddings()(A )
                      lowercase  :		int						=      model.get_input_embeddings()(A )

               lowercase  :		int						=      self._prepare_for_class(A   ,		A )

               @tf.function
               def run_in_graph_mode():
                      return model(A )

               lowercase  :		int						=      run_in_graph_mode()
               self.assertIsNotNone(A )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      []
        if model.config.bos_token_id is not None:
               special_tokens.append(model.config.bos_token_id )
        if model.config.pad_token_id is not None:
               special_tokens.append(model.config.pad_token_id )
        if model.config.eos_token_id is not None:
               special_tokens.append(model.config.eos_token_id )

        # create random bad tokens that are not special tokens
        lowercase  :		int						=      []
        while len(A ) < num_bad_tokens:
               lowercase  :		List[Any]						=      tf.squeeze(ids_tensor((1, 1)   ,		self.model_tester.vocab_size )   ,		0 ).numpy()[0]
               if token not in special_tokens:
                      bad_tokens.append(A )
        return bad_tokens



 def    SCREAMING_SNAKE_CASE(  self   ,		A ):






        '''simple docstring'''




        for token_id in output_ids[0].numpy().tolist():
               self.assertGreaterEqual(A   ,		0 )
               self.assertLess(A   ,		self.model_tester.vocab_size )





 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A ):






        '''simple docstring'''




        for bad_word_ids in bad_words_ids:
               # for all slices in batch
               for generated_ids_slice in generated_ids:
                      # for all word idx
                      for i in range(len(A )   ,		len(A ) ):
                             # if tokens match
                             if generated_ids_slice[i - len(A ) : i] == bad_word_ids:
                                    return True
        return False










'''simple docstring'''






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece

class   SCREAMING_SNAKE_CASE_		(       unittest.TestCase				):



 def    SCREAMING_SNAKE_CASE(  self   ,		A ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      list(tokenizer_class.max_model_input_sizes.keys() )
        for model_name in sa_models[:1]:
               lowercase  :		Dict						=      tokenizer_class.from_pretrained(A )
               self.assertIsNotNone(A )
               self.assertIsInstance(A   ,		A )
               self.assertIsInstance(A   ,		A )

               for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(A   ,		A )
                      lowercase  :		Tuple						=      tokenizer.convert_tokens_to_ids(A )
                      self.assertIsInstance(A   ,		A )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A = None ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      pickle.dumps(A )
        self.assertIsNotNone(A )

        lowercase  :		str						=      pickle.loads(A )

        # Ensure is_fast is correctly restored
        self.assertEqual(be_restored.is_fast   ,		be_original.is_fast )

        # Ensure encodings are potentially correctly restored
        if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings )
        else:
               self.assertIsNone(be_restored.encodings )

        # Ensure the keys are the same
        for original_v, restored_v in zip(be_original.values()   ,		be_restored.values() ):
               if equal_op:
                      self.assertTrue(equal_op(A   ,		A ) )
               else:
                      self.assertEqual(A   ,		A )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.check_tokenizer_from_pretrained(A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.assertEqual(TensorType('''tf''' )   ,		TensorType.TENSORFLOW )
        self.assertEqual(TensorType('''pt''' )   ,		TensorType.PYTORCH )
        self.assertEqual(TensorType('''np''' )   ,		TensorType.NUMPY )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import numpy as np

        lowercase  :		Union[str, Any]						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		str						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        # Python no tensor
        with self.subTest('''BatchEncoding (Python, return_tensors=None)''' ):
               self.assert_dump_and_restore(tokenizer_p('''Small example to encode''' ) )

        with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.NUMPY )   ,		np.array_equal )

        with self.subTest('''BatchEncoding (Rust, return_tensors=None)''' ):
               self.assert_dump_and_restore(tokenizer_r('''Small example to encode''' ) )

        with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.NUMPY )   ,		np.array_equal )



 @require_tf
 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import tensorflow as tf

        def tf_array_equals(A   ,		A ):
               return tf.reduce_all(tf.equal(A   ,		A ) )

        lowercase  :		Optional[Any]						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		Optional[Any]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.TENSORFLOW )   ,		A )

        with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.TENSORFLOW )   ,		A )



 @require_torch
 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import torch

        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		Any						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.PYTORCH )   ,		torch.equal )

        with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.PYTORCH )   ,		torch.equal )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		str						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		List[Any]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''Python Tokenizer''' ):
               self.assertFalse(tokenizer_p('''Small example to_encode''' ).is_fast )

        with self.subTest('''Rust Tokenizer''' ):
               self.assertTrue(tokenizer_r('''Small example to_encode''' ).is_fast )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )
        lowercase  :		str						=      tokenizer_r(['''Test''', '''\xad''', '''test''']   ,		is_split_into_words=A )

        self.assertEqual(encoded.word_to_tokens(0 )   ,		TokenSpan(start=1   ,		end=2 ) )
        self.assertEqual(encoded.word_to_tokens(1 )   ,		A )
        self.assertEqual(encoded.word_to_tokens(2 )   ,		TokenSpan(start=2   ,		end=3 ) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''np''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Any						=      batch.convert_to_tensors(tensor_type='''np''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''np'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_torch
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		Union[str, Any]						=      batch.convert_to_tensors(tensor_type='''pt''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Tuple						=      batch.convert_to_tensors(tensor_type='''pt''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		Optional[Any]						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''pt'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_tf
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		Dict						=      batch.convert_to_tensors(tensor_type='''tf''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Optional[int]						=      batch.convert_to_tensors(tensor_type='''tf''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		str						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''tf'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_flax
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Tuple						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		List[Any]						=      batch.convert_to_tensors(tensor_type='''jax''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Dict						=      batch.convert_to_tensors(tensor_type='''jax''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		str						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		str						=      batch.convert_to_tensors(tensor_type='''jax'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      [{'''input_ids''': np.array([0, 1, 2] )}, {'''input_ids''': np.array([0, 1, 2, 3] )}]
        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		Dict						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		np.ndarray ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Dict						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''np''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		np.ndarray ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_torch
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import torch

        lowercase  :		Optional[Any]						=      [{'''input_ids''': torch.tensor([0, 1, 2] )}, {'''input_ids''': torch.tensor([0, 1, 2, 3] )}]
        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		Any						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		torch.Tensor ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Union[str, Any]						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''pt''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		torch.Tensor ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_tf
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import tensorflow as tf

        lowercase  :		Optional[int]						=      [{'''input_ids''': tf.constant([0, 1, 2] )}, {'''input_ids''': tf.constant([0, 1, 2, 3] )}]
        lowercase  :		Dict						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		List[Any]						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		tf.Tensor ) )
        self.assertEqual(batch['''input_ids'''].numpy().tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Any						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''tf''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		tf.Tensor ) )
        self.assertEqual(batch['''input_ids'''].numpy().tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
        PreTrainedTokenizerFast(tokenizer_object=A )





 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
        with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(A   ,		'''tokenizer.json''' ) )
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(A   ,		'''tokenizer.json''' ) )




