




'''simple docstring'''






from __future__ import annotations

import unittest

from transformers import AlbertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
      from transformers.models.albert.modeling_tf_albert import (
          TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFAlbertForMaskedLM,
          TFAlbertForMultipleChoice,
          TFAlbertForPreTraining,
          TFAlbertForQuestionAnswering,
          TFAlbertForSequenceClassification,
          TFAlbertForTokenClassification,
          TFAlbertModel,
      )

class   SCREAMING_SNAKE_CASE_		:



 def __init__(  self   ,		A   ,		A=13   ,		A=7   ,		A=True   ,		A=True   ,		A=True   ,		A=True   ,		A=99   ,		A=16   ,		A=32   ,		A=2   ,		A=4   ,		A=37   ,		A="gelu"   ,		A=0.1   ,		A=0.1   ,		A=512   ,		A=16   ,		A=2   ,		A=0.0_2   ,		A=3   ,		A=4   ,		A=None   ,		):






        '''simple docstring'''




        lowercase  :		Dict						=      parent
        lowercase  :		Any						=      13
        lowercase  :		str						=      7
        lowercase  :		Optional[Any]						=      True
        lowercase  :		Any						=      True
        lowercase  :		Optional[int]						=      True
        lowercase  :		str						=      True
        lowercase  :		int						=      99
        lowercase  :		int						=      16
        lowercase  :		str						=      32
        lowercase  :		Tuple						=      2
        lowercase  :		int						=      4
        lowercase  :		int						=      37
        lowercase  :		List[str]						=      '''gelu'''
        lowercase  :		List[str]						=      0.1
        lowercase  :		Union[str, Any]						=      0.1
        lowercase  :		Any						=      512
        lowercase  :		int						=      16
        lowercase  :		Optional[Any]						=      2
        lowercase  :		Any						=      0.0_2
        lowercase  :		List[Any]						=      3
        lowercase  :		int						=      4
        lowercase  :		Tuple						=      None



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size )

        lowercase  :		List[str]						=      None
        if self.use_input_mask:
               lowercase  :		Union[str, Any]						=      random_attention_mask([self.batch_size, self.seq_length] )

        lowercase  :		Tuple						=      None
        if self.use_token_type_ids:
               lowercase  :		Optional[int]						=      ids_tensor([self.batch_size, self.seq_length]   ,		self.type_vocab_size )

        lowercase  :		int						=      None
        lowercase  :		Tuple						=      None
        lowercase  :		int						=      None
        if self.use_labels:
               lowercase  :		Optional[int]						=      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size )
               lowercase  :		Optional[Any]						=      ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels )
               lowercase  :		List[str]						=      ids_tensor([self.batch_size]   ,		self.num_choices )

        lowercase  :		str						=      AlbertConfig(
            vocab_size=self.vocab_size   ,		hidden_size=self.hidden_size   ,		embedding_size=self.embedding_size   ,		num_hidden_layers=self.num_hidden_layers   ,		num_attention_heads=self.num_attention_heads   ,		intermediate_size=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		hidden_dropout_prob=self.hidden_dropout_prob   ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,		max_position_embeddings=self.max_position_embeddings   ,		type_vocab_size=self.type_vocab_size   ,		initializer_range=self.initializer_range   ,		)

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      TFAlbertModel(config=A )
        # inputs = {'input_ids': input_ids,
        #           'attention_mask': input_mask,
        #           'token_type_ids': token_type_ids}
        # sequence_output, pooled_output = model(**inputs)
        lowercase  :		List[Any]						=      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        lowercase  :		Union[str, Any]						=      model(A )

        lowercase  :		Optional[Any]						=      [input_ids, input_mask]
        lowercase  :		Union[str, Any]						=      model(A )

        lowercase  :		Dict						=      model(A )

        self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size) )
        self.parent.assertEqual(result.pooler_output.shape   ,		(self.batch_size, self.hidden_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      self.num_labels
        lowercase  :		Optional[int]						=      TFAlbertForPreTraining(config=A )
        lowercase  :		List[str]						=      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        lowercase  :		str						=      model(A )
        self.parent.assertEqual(result.prediction_logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size) )
        self.parent.assertEqual(result.sop_logits.shape   ,		(self.batch_size, self.num_labels) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      TFAlbertForMaskedLM(config=A )
        lowercase  :		Any						=      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        lowercase  :		Dict						=      model(A )
        self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		str						=      self.num_labels
        lowercase  :		str						=      TFAlbertForSequenceClassification(config=A )
        lowercase  :		List[str]						=      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        lowercase  :		Any						=      model(A )
        self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_labels) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		str						=      TFAlbertForQuestionAnswering(config=A )
        lowercase  :		Optional[int]						=      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        lowercase  :		Union[str, Any]						=      model(A )
        self.parent.assertEqual(result.start_logits.shape   ,		(self.batch_size, self.seq_length) )
        self.parent.assertEqual(result.end_logits.shape   ,		(self.batch_size, self.seq_length) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		str						=      self.num_choices
        lowercase  :		Optional[Any]						=      TFAlbertForMultipleChoice(config=A )
        lowercase  :		Any						=      tf.tile(tf.expand_dims(A   ,		1 )   ,		(1, self.num_choices, 1) )
        lowercase  :		Union[str, Any]						=      tf.tile(tf.expand_dims(A   ,		1 )   ,		(1, self.num_choices, 1) )
        lowercase  :		Any						=      tf.tile(tf.expand_dims(A   ,		1 )   ,		(1, self.num_choices, 1) )
        lowercase  :		Optional[Any]						=      {
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
            '''token_type_ids''': multiple_choice_token_type_ids,
        }
        lowercase  :		Union[str, Any]						=      model(A )
        self.parent.assertListEqual(list(result['''logits'''].shape )   ,		[self.batch_size, self.num_choices] )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      self.num_labels
        lowercase  :		List[str]						=      TFAlbertForTokenClassification(config=A )
        lowercase  :		Dict						=      {
            '''input_ids''': input_ids,
            '''attention_mask''': input_mask,
            '''token_type_ids''': token_type_ids,
        }
        lowercase  :		Any						=      model(A )
        self.parent.assertListEqual(list(result['''logits'''].shape )   ,		[self.batch_size, self.seq_length, self.num_labels] )





 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      self.prepare_config_and_inputs()
        (
            (
            lowercase
        ),     (
            lowercase
        ),     (
            lowercase
        ),     (
            lowercase
        ),     (
            lowercase
        ),     (
            lowercase
        ),     (
            lowercase
        ),     
        )  :		Optional[int]						=      config_and_inputs
        lowercase  :		str						=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict






@require_tf
class   SCREAMING_SNAKE_CASE_		(       lowerCamelCase_		,     lowerCamelCase_		,     unittest.TestCase				):
 __snake_case	:     Optional[int]			   =      (
     (
         TFAlbertModel,
         TFAlbertForPreTraining,
         TFAlbertForMaskedLM,
         TFAlbertForSequenceClassification,
         TFAlbertForQuestionAnswering,
         TFAlbertForTokenClassification,
         TFAlbertForMultipleChoice,
     )
     if is_tf_available()
     else ()
 )
 __snake_case	:     Tuple			   =      (
     {
         "feature-extraction": TFAlbertModel,
         "fill-mask": TFAlbertForMaskedLM,
         "question-answering": TFAlbertForQuestionAnswering,
         "text-classification": TFAlbertForSequenceClassification,
         "token-classification": TFAlbertForTokenClassification,
         "zero-shot": TFAlbertForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 __snake_case	:     List[str]			   =      False
 __snake_case	:     Dict			   =      False



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A=False ):






        '''simple docstring'''




        lowercase  :		Any						=      super()._prepare_for_class(A   ,		A   ,		return_labels=A )

        if return_labels:
               if model_class in get_values(A ):
                      lowercase  :		int						=      tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa )

        return inputs_dict



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      TFAlbertModelTester(self )
        lowercase  :		str						=      ConfigTester(self   ,		config_class=A   ,		hidden_size=37 )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.config_tester.run_common_tests()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		str						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_model(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_for_pretraining(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_for_masked_lm(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_for_multiple_choice(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		int						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_for_sequence_classification(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_albert_for_question_answering(*A )





 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase  :		Tuple						=      TFAlbertModel.from_pretrained(A )
               self.assertIsNotNone(A )






@require_tf
class   SCREAMING_SNAKE_CASE_		(       unittest.TestCase				):



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		str						=      TFAlbertForPreTraining.from_pretrained('''albert-base-v2''' )
        lowercase  :		Optional[int]						=      tf.constant([[0, 1, 2, 3, 4, 5]] )
        lowercase  :		Any						=      model(A )[0]

        lowercase  :		List[Any]						=      [1, 6, 30000]
        self.assertEqual(output.shape   ,		A )

        lowercase  :		Optional[Any]						=      tf.constant(
            [
                [
                    [4.5_9_5_6_6_8, 0.7_4_4_6_2_7_5_4, -1.8_1_8_1_4_7],
                    [4.5_9_5_4_3_4_7, 0.7_4_5_4_1_8_4, -1.8_1_8_8_2_5_8],
                    [4.5_9_5_4_9_0_5, 0.7_4_4_8_2_3_5, -1.8_1_8_2_3_1_6],
                ]
            ] )
        tf.debugging.assert_near(output[:, :3, :3]   ,		A   ,		atol=1E-4 )










'''simple docstring'''






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece

class   SCREAMING_SNAKE_CASE_		(       unittest.TestCase				):



 def    SCREAMING_SNAKE_CASE(  self   ,		A ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      list(tokenizer_class.max_model_input_sizes.keys() )
        for model_name in sa_models[:1]:
               lowercase  :		Dict						=      tokenizer_class.from_pretrained(A )
               self.assertIsNotNone(A )
               self.assertIsInstance(A   ,		A )
               self.assertIsInstance(A   ,		A )

               for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(A   ,		A )
                      lowercase  :		Tuple						=      tokenizer.convert_tokens_to_ids(A )
                      self.assertIsInstance(A   ,		A )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A = None ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      pickle.dumps(A )
        self.assertIsNotNone(A )

        lowercase  :		str						=      pickle.loads(A )

        # Ensure is_fast is correctly restored
        self.assertEqual(be_restored.is_fast   ,		be_original.is_fast )

        # Ensure encodings are potentially correctly restored
        if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings )
        else:
               self.assertIsNone(be_restored.encodings )

        # Ensure the keys are the same
        for original_v, restored_v in zip(be_original.values()   ,		be_restored.values() ):
               if equal_op:
                      self.assertTrue(equal_op(A   ,		A ) )
               else:
                      self.assertEqual(A   ,		A )



 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.check_tokenizer_from_pretrained(A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.assertEqual(TensorType('''tf''' )   ,		TensorType.TENSORFLOW )
        self.assertEqual(TensorType('''pt''' )   ,		TensorType.PYTORCH )
        self.assertEqual(TensorType('''np''' )   ,		TensorType.NUMPY )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import numpy as np

        lowercase  :		Union[str, Any]						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		str						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        # Python no tensor
        with self.subTest('''BatchEncoding (Python, return_tensors=None)''' ):
               self.assert_dump_and_restore(tokenizer_p('''Small example to encode''' ) )

        with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.NUMPY )   ,		np.array_equal )

        with self.subTest('''BatchEncoding (Rust, return_tensors=None)''' ):
               self.assert_dump_and_restore(tokenizer_r('''Small example to encode''' ) )

        with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.NUMPY )   ,		np.array_equal )



 @require_tf
 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import tensorflow as tf

        def tf_array_equals(A   ,		A ):
               return tf.reduce_all(tf.equal(A   ,		A ) )

        lowercase  :		Optional[Any]						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		Optional[Any]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.TENSORFLOW )   ,		A )

        with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.TENSORFLOW )   ,		A )



 @require_torch
 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import torch

        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		Any						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)''' ):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''   ,		return_tensors=TensorType.PYTORCH )   ,		torch.equal )

        with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)''' ):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''   ,		return_tensors=TensorType.PYTORCH )   ,		torch.equal )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		str						=      BertTokenizer.from_pretrained('''bert-base-cased''' )
        lowercase  :		List[Any]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )

        with self.subTest('''Python Tokenizer''' ):
               self.assertFalse(tokenizer_p('''Small example to_encode''' ).is_fast )

        with self.subTest('''Rust Tokenizer''' ):
               self.assertTrue(tokenizer_r('''Small example to_encode''' ).is_fast )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      BertTokenizerFast.from_pretrained('''bert-base-cased''' )
        lowercase  :		str						=      tokenizer_r(['''Test''', '''\xad''', '''test''']   ,		is_split_into_words=A )

        self.assertEqual(encoded.word_to_tokens(0 )   ,		TokenSpan(start=1   ,		end=2 ) )
        self.assertEqual(encoded.word_to_tokens(1 )   ,		A )
        self.assertEqual(encoded.word_to_tokens(2 )   ,		TokenSpan(start=2   ,		end=3 ) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''np''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Any						=      batch.convert_to_tensors(tensor_type='''np''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''np'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_torch
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		Union[str, Any]						=      batch.convert_to_tensors(tensor_type='''pt''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Tuple						=      batch.convert_to_tensors(tensor_type='''pt''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		Optional[Any]						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''pt'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_tf
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		Dict						=      batch.convert_to_tensors(tensor_type='''tf''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Optional[int]						=      batch.convert_to_tensors(tensor_type='''tf''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		str						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		List[str]						=      batch.convert_to_tensors(tensor_type='''tf'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 @require_flax
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Tuple						=      BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
        lowercase  :		List[Any]						=      batch.convert_to_tensors(tensor_type='''jax''' )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(2, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(2,) )
        # test converting the converted
        with CaptureStderr() as cs:
               lowercase  :		Dict						=      batch.convert_to_tensors(tensor_type='''jax''' )
        self.assertFalse(len(cs.err )   ,		msg=f"""should have no warning, but got {cs.err}""" )

        lowercase  :		str						=      BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
        lowercase  :		str						=      batch.convert_to_tensors(tensor_type='''jax'''   ,		prepend_batch_axis=A )
        self.assertEqual(tensor_batch['''inputs'''].shape   ,		(1, 3) )
        self.assertEqual(tensor_batch['''labels'''].shape   ,		(1,) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      [{'''input_ids''': np.array([0, 1, 2] )}, {'''input_ids''': np.array([0, 1, 2, 3] )}]
        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		Dict						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		np.ndarray ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Dict						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''np''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		np.ndarray ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_torch
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import torch

        lowercase  :		Optional[Any]						=      [{'''input_ids''': torch.tensor([0, 1, 2] )}, {'''input_ids''': torch.tensor([0, 1, 2, 3] )}]
        lowercase  :		Tuple						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		Any						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		torch.Tensor ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Union[str, Any]						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''pt''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		torch.Tensor ) )
        self.assertEqual(batch['''input_ids'''].tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_tf
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        import tensorflow as tf

        lowercase  :		Optional[int]						=      [{'''input_ids''': tf.constant([0, 1, 2] )}, {'''input_ids''': tf.constant([0, 1, 2, 3] )}]
        lowercase  :		Dict						=      BertTokenizer.from_pretrained('''bert-base-cased''' )

        lowercase  :		List[Any]						=      tokenizer.pad(A   ,		padding=A )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		tf.Tensor ) )
        self.assertEqual(batch['''input_ids'''].numpy().tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
        lowercase  :		Any						=      tokenizer.pad(A   ,		padding=A   ,		return_tensors='''tf''' )
        self.assertTrue(isinstance(batch['''input_ids''']   ,		tf.Tensor ) )
        self.assertEqual(batch['''input_ids'''].numpy().tolist()   ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )



 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
        PreTrainedTokenizerFast(tokenizer_object=A )





 @require_tokenizers
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
        with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(A   ,		'''tokenizer.json''' ) )
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(A   ,		'''tokenizer.json''' ) )




