






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )






class 	lowercase_      :






      '''simple docstring'''




      def __init__(   self   :		List[Any]				,      __UpperCAmelCase   :		Union[str, Any]				,      )					->Tuple:




       """simple docstring"""


       a							=   parent
       a							=   13
       a							=   7
       a							=   True
       a							=   True
       a							=   True
       a							=   True
       a							=   True
       a							=   False
       a							=   False
       a							=   False
       a							=   2
       a							=   99
       a							=   0
       a							=   32
       a							=   2
       a							=   4
       a							=   0.1
       a							=   0.1
       a							=   512
       a							=   16
       a							=   2
       a							=   0.02
       a							=   3
       a							=   4
       a							=   '''last'''
       a							=   True
       a							=   None
       a							=   0







      def    __lowerCAmelCase      (   self   :		int						)					->Union[str, Any]:




       """simple docstring"""


       a							=   ids_tensor([self.batch_size, self.seq_length]				,      self.vocab_size						)
       a							=   random_attention_mask([self.batch_size, self.seq_length]				,      dtype=tf.floataa						)

       a							=   None
       if self.use_input_lengths:
        a							=   (
            ids_tensor([self.batch_size]				,      vocab_size=2						) + self.seq_length - 2
        )  # small variation of seq_length

       a							=   None
       if self.use_token_type_ids:
        a							=   ids_tensor([self.batch_size, self.seq_length]				,      self.n_langs						)

       a							=   None
       a							=   None
       a							=   None
       if self.use_labels:
        a							=   ids_tensor([self.batch_size]				,      self.type_sequence_label_size						)
        a							=   ids_tensor([self.batch_size, self.seq_length]				,      self.num_labels						)
        a							=   ids_tensor([self.batch_size]				,      2				,      dtype=tf.floataa						)
        a							=   ids_tensor([self.batch_size]				,      self.num_choices						)

       a							=   XLMConfig(
           vocab_size=self.vocab_size				,      n_special=self.n_special				,      emb_dim=self.hidden_size				,      n_layers=self.num_hidden_layers				,      n_heads=self.num_attention_heads				,      dropout=self.hidden_dropout_prob				,      attention_dropout=self.attention_probs_dropout_prob				,      gelu_activation=self.gelu_activation				,      sinusoidal_embeddings=self.sinusoidal_embeddings				,      asm=self.asm				,      causal=self.causal				,      n_langs=self.n_langs				,      max_position_embeddings=self.max_position_embeddings				,      initializer_range=self.initializer_range				,      summary_type=self.summary_type				,      use_proj=self.use_proj				,      bos_token_id=self.bos_token_id				,      )

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )







      def    __lowerCAmelCase      (   self   :		Union[str, Any]				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		List[str]				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		str				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Any				,      __UpperCAmelCase   :		str				,      )					->Union[str, Any]:




       """simple docstring"""


       a							=   TFXLMModel(config=__UpperCAmelCase						)
       a							=   {'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
       a							=   model(__UpperCAmelCase						)

       a							=   [input_ids, input_mask]
       a							=   model(__UpperCAmelCase						)
       self.parent.assertEqual(result.last_hidden_state.shape				,      (self.batch_size, self.seq_length, self.hidden_size)						)







      def    __lowerCAmelCase      (   self   :		List[str]				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		List[str]				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		List[Any]				,      __UpperCAmelCase   :		Optional[Any]				,      )					->Optional[Any]:




       """simple docstring"""


       a							=   TFXLMWithLMHeadModel(__UpperCAmelCase						)

       a							=   {'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
       a							=   model(__UpperCAmelCase						)

       a							=   outputs

       self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.seq_length, self.vocab_size)						)







      def    __lowerCAmelCase      (   self   :		Optional[int]				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		List[str]				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		List[str]				,      )					->Optional[Any]:




       """simple docstring"""


       a							=   TFXLMForQuestionAnsweringSimple(__UpperCAmelCase						)

       a							=   {'''input_ids''': input_ids, '''lengths''': input_lengths}

       a							=   model(__UpperCAmelCase						)

       self.parent.assertEqual(result.start_logits.shape				,      (self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape				,      (self.batch_size, self.seq_length)						)







      def    __lowerCAmelCase      (   self   :		Optional[Any]				,      __UpperCAmelCase   :		int				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		str				,      __UpperCAmelCase   :		str				,      __UpperCAmelCase   :		int				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		str				,      __UpperCAmelCase   :		Optional[int]				,      )					->List[str]:




       """simple docstring"""


       a							=   TFXLMForSequenceClassification(__UpperCAmelCase						)

       a							=   {'''input_ids''': input_ids, '''lengths''': input_lengths}

       a							=   model(__UpperCAmelCase						)

       self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.type_sequence_label_size)						)







      def    __lowerCAmelCase      (   self   :		int				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		str				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		str				,      )					->Optional[int]:




       """simple docstring"""


       a							=   self.num_labels
       a							=   TFXLMForTokenClassification(config=__UpperCAmelCase						)
       a							=   {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
       a							=   model(__UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.seq_length, self.num_labels)						)







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		Any				,      __UpperCAmelCase   :		Optional[Any]				,      __UpperCAmelCase   :		List[str]				,      __UpperCAmelCase   :		List[Any]				,      __UpperCAmelCase   :		List[str]				,      __UpperCAmelCase   :		Tuple				,      __UpperCAmelCase   :		List[str]				,      )					->int:




       """simple docstring"""


       a							=   self.num_choices
       a							=   TFXLMForMultipleChoice(config=__UpperCAmelCase						)
       a							=   tf.tile(tf.expand_dims(__UpperCAmelCase				,      1						)				,      (1, self.num_choices, 1)						)
       a							=   tf.tile(tf.expand_dims(__UpperCAmelCase				,      1						)				,      (1, self.num_choices, 1)						)
       a							=   tf.tile(tf.expand_dims(__UpperCAmelCase				,      1						)				,      (1, self.num_choices, 1)						)
       a							=   {
           '''input_ids''': multiple_choice_inputs_ids,
           '''attention_mask''': multiple_choice_input_mask,
           '''token_type_ids''': multiple_choice_token_type_ids,
       }
       a							=   model(__UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.num_choices)						)







      def    __lowerCAmelCase      (   self   :		Dict						)					->str:




       """simple docstring"""


       a							=   self.prepare_config_and_inputs()
       (
           (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,  (
           a  
       )						,    
       )							=   config_and_inputs
       a							=   {
           '''input_ids''': input_ids,
           '''token_type_ids''': token_type_ids,
           '''langs''': token_type_ids,
           '''lengths''': input_lengths,
       }
       return config, inputs_dict






@require_tf
class 	lowercase_      (			lowercase				,						lowercase				,						unittest.TestCase     ):






      '''simple docstring'''




      __snake_case  						=						(
          (
              TFXLMModel,
              TFXLMWithLMHeadModel,
              TFXLMForSequenceClassification,
              TFXLMForQuestionAnsweringSimple,
              TFXLMForTokenClassification,
              TFXLMForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      __snake_case  						=						(
          (TFXLMWithLMHeadModel,) if is_tf_available() else ()
      )  # TODO (PVP): Check other models whether language generation is also applicable
      __snake_case  						=						(
          {
              '''feature-extraction''': TFXLMModel,
              '''fill-mask''': TFXLMWithLMHeadModel,
              '''question-answering''': TFXLMForQuestionAnsweringSimple,
              '''text-classification''': TFXLMForSequenceClassification,
              '''text-generation''': TFXLMWithLMHeadModel,
              '''token-classification''': TFXLMForTokenClassification,
              '''zero-shot''': TFXLMForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      __snake_case  						=						False
      __snake_case  						=						False







      def    __lowerCAmelCase      (   self   :		List[str]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		int				,      __UpperCAmelCase   :		List[Any]				,      __UpperCAmelCase   :		Optional[int]				,      __UpperCAmelCase   :		Tuple						)					->List[Any]:




       """simple docstring"""


       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('''Fast'''						)
       ):
        # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
        # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
        # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
        return True

       return False







      def    __lowerCAmelCase      (   self   :		List[Any]						)					->List[Any]:




       """simple docstring"""


       a							=   TFXLMModelTester(self						)
       a							=   ConfigTester(self				,      config_class=__UpperCAmelCase				,      emb_dim=37						)







      def    __lowerCAmelCase      (   self   :		List[Any]						)					->List[Any]:




       """simple docstring"""


       self.config_tester.run_common_tests()







      def    __lowerCAmelCase      (   self   :		Dict						)					->Optional[int]:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		Union[str, Any]						)					->List[str]:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		Tuple						)					->List[Any]:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		Optional[Any]						)					->Optional[Any]:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		List[Any]						)					->Tuple:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_token_classification(*__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		Optional[int]						)					->int:




       """simple docstring"""


       a							=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*__UpperCAmelCase						)







      @slow
      def    __lowerCAmelCase      (   self   :		Optional[Any]						)					->Optional[int]:




       """simple docstring"""


       for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        a							=   TFXLMModel.from_pretrained(__UpperCAmelCase						)
        self.assertIsNotNone(__UpperCAmelCase						)






@require_tf
class 	lowercase_      (			unittest.TestCase     ):






      '''simple docstring'''




      @slow
      def    __lowerCAmelCase      (   self   :		Union[str, Any]						)					->Union[str, Any]:




       """simple docstring"""


       a							=   TFXLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''						)
       a							=   tf.convert_to_tensor([[14, 447]]				,      dtype=tf.intaa						)  # the president
       a							=   [
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       a							=   model.generate(__UpperCAmelCase				,      do_sample=__UpperCAmelCase						)
       self.assertListEqual(output_ids[0].numpy().tolist()				,      __UpperCAmelCase						)










import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 	lowercase_      (			lowercase     ):






      '''simple docstring'''




      __snake_case  						=						['''image_processor''', '''tokenizer''']
      __snake_case  						=						'''AutoImageProcessor'''
      __snake_case  						=						'''AutoTokenizer'''







      def __init__(   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]=None				,      __UpperCAmelCase   :		int=None				,      **__UpperCAmelCase   :		Union[str, Any]						)					->Dict:




       """simple docstring"""


       a							=   None
       if "feature_extractor" in kwargs:
        warnings.warn(
            '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
            ''' instead.'''				,      __UpperCAmelCase				,      )
        a							=   kwargs.pop('''feature_extractor'''						)

       a							=   image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
        raise ValueError('''You have to specify an image_processor.'''						)
       if tokenizer is None:
        raise ValueError('''You have to specify a tokenizer.'''						)

       super().__init__(__UpperCAmelCase				,      __UpperCAmelCase						)
       a							=   self.image_processor







      def __call__(   self   :		List[Any]				,      __UpperCAmelCase   :		Optional[Any]=None				,      __UpperCAmelCase   :		Optional[int]=None				,      __UpperCAmelCase   :		str=None				,      **__UpperCAmelCase   :		List[Any]						)					->Dict:




       """simple docstring"""


       if text is None and images is None:
        raise ValueError('''You have to specify either text or images. Both cannot be none.'''						)

       if text is not None:
        a							=   self.tokenizer(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if images is not None:
        a							=   self.image_processor(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if text is not None and images is not None:
        a							=   image_features.pixel_values
        return encoding
       elif text is not None:
        return encoding
       else:
        return BatchEncoding(data=dict(**__UpperCAmelCase						)				,      tensor_type=__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		List[Any]				,      *__UpperCAmelCase   :		Optional[Any]				,      **__UpperCAmelCase   :		List[str]						)					->Optional[Any]:




       """simple docstring"""


       return self.tokenizer.batch_decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		str				,      *__UpperCAmelCase   :		List[Any]				,      **__UpperCAmelCase   :		Optional[Any]						)					->Union[str, Any]:




       """simple docstring"""


       return self.tokenizer.decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->str:




       """simple docstring"""


       a							=   self.tokenizer.model_input_names
       a							=   self.image_processor.model_input_names
       return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names						)						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor_class







      @property
      def    __lowerCAmelCase      (   self   :		int						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor


