






'''simple docstring'''


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )




class 							lowerCAmelCase					:







   """simple docstring"""







   def __init__( self   ,   _A   ,   )    ->   List[str]:
    __a    :	Dict									=						parent
    __a    :	Union[str, Any]									=						13
    __a    :	str									=						7
    __a    :	List[Any]									=						True
    __a    :	Any									=						True
    __a    :	Union[str, Any]									=						True
    __a    :	Dict									=						True
    __a    :	Tuple									=						True
    __a    :	Any									=						False
    __a    :	List[str]									=						False
    __a    :	Optional[Any]									=						False
    __a    :	Optional[Any]									=						2
    __a    :	List[str]									=						99
    __a    :	Optional[int]									=						0
    __a    :	Dict									=						32
    __a    :	str									=						2
    __a    :	List[str]									=						4
    __a    :	str									=						0.1
    __a    :	str									=						0.1
    __a    :	List[Any]									=						512
    __a    :	Optional[int]									=						16
    __a    :	Tuple									=						2
    __a    :	Any									=						0.02
    __a    :	List[str]									=						3
    __a    :	Dict									=						4
    __a    :	Union[str, Any]									=						'last'
    __a    :	Optional[Any]									=						True
    __a    :	Optional[Any]									=						None
    __a    :	List[str]									=						0



   def    __magic_name__    ( self			)    ->   str:
    __a    :	str									=						ids_tensor([self.batch_size, self.seq_length]   ,   self.vocab_size			)
    __a    :	List[Any]									=						random_attention_mask([self.batch_size, self.seq_length]   ,   dtype=tf.floataa			)

    __a    :	Tuple									=						None
    if self.use_input_lengths:
     __a    :	Optional[Any]									=						(
         ids_tensor([self.batch_size]   ,   vocab_size=2			) + self.seq_length - 2
     )  # small variation of seq_length

    __a    :	Dict									=						None
    if self.use_token_type_ids:
     __a    :	List[str]									=						ids_tensor([self.batch_size, self.seq_length]   ,   self.n_langs			)

    __a    :	Optional[int]									=						None
    __a    :	Dict									=						None
    __a    :	Any									=						None
    if self.use_labels:
     __a    :	Optional[Any]									=						ids_tensor([self.batch_size]   ,   self.type_sequence_label_size			)
     __a    :	Any									=						ids_tensor([self.batch_size, self.seq_length]   ,   self.num_labels			)
     __a    :	Optional[Any]									=						ids_tensor([self.batch_size]   ,   2   ,   dtype=tf.floataa			)
     __a    :	Any									=						ids_tensor([self.batch_size]   ,   self.num_choices			)

    __a    :	Union[str, Any]									=						XLMConfig(
        vocab_size=self.vocab_size   ,   n_special=self.n_special   ,   emb_dim=self.hidden_size   ,   n_layers=self.num_hidden_layers   ,   n_heads=self.num_attention_heads   ,   dropout=self.hidden_dropout_prob   ,   attention_dropout=self.attention_probs_dropout_prob   ,   gelu_activation=self.gelu_activation   ,   sinusoidal_embeddings=self.sinusoidal_embeddings   ,   asm=self.asm   ,   causal=self.causal   ,   n_langs=self.n_langs   ,   max_position_embeddings=self.max_position_embeddings   ,   initializer_range=self.initializer_range   ,   summary_type=self.summary_type   ,   use_proj=self.use_proj   ,   bos_token_id=self.bos_token_id   ,   )

    return (
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
        choice_labels,
        input_mask,
    )



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   List[Any]:
    __a    :	Union[str, Any]									=						TFXLMModel(config=_A			)
    __a    :	int									=						{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
    __a    :	Any									=						model(_A			)

    __a    :	Optional[Any]									=						[input_ids, input_mask]
    __a    :	Any									=						model(_A			)
    self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, self.seq_length, self.hidden_size)			)



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   Any:
    __a    :	Optional[Any]									=						TFXLMWithLMHeadModel(_A			)

    __a    :	List[str]									=						{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
    __a    :	Union[str, Any]									=						model(_A			)

    __a    :	Dict									=						outputs

    self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.seq_length, self.vocab_size)			)



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   Any:
    __a    :	Optional[int]									=						TFXLMForQuestionAnsweringSimple(_A			)

    __a    :	List[Any]									=						{'input_ids': input_ids, 'lengths': input_lengths}

    __a    :	str									=						model(_A			)

    self.parent.assertEqual(result.start_logits.shape   ,   (self.batch_size, self.seq_length)			)
    self.parent.assertEqual(result.end_logits.shape   ,   (self.batch_size, self.seq_length)			)



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   Union[str, Any]:
    __a    :	List[Any]									=						TFXLMForSequenceClassification(_A			)

    __a    :	Dict									=						{'input_ids': input_ids, 'lengths': input_lengths}

    __a    :	int									=						model(_A			)

    self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.type_sequence_label_size)			)



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   Tuple:
    __a    :	List[str]									=						self.num_labels
    __a    :	List[str]									=						TFXLMForTokenClassification(config=_A			)
    __a    :	Tuple									=						{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
    __a    :	Any									=						model(_A			)
    self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.seq_length, self.num_labels)			)



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A   ,   )    ->   Dict:
    __a    :	Any									=						self.num_choices
    __a    :	Any									=						TFXLMForMultipleChoice(config=_A			)
    __a    :	Union[str, Any]									=						tf.tile(tf.expand_dims(_A   ,   1			)   ,   (1, self.num_choices, 1)			)
    __a    :	Any									=						tf.tile(tf.expand_dims(_A   ,   1			)   ,   (1, self.num_choices, 1)			)
    __a    :	Tuple									=						tf.tile(tf.expand_dims(_A   ,   1			)   ,   (1, self.num_choices, 1)			)
    __a    :	Union[str, Any]									=						{
        'input_ids': multiple_choice_inputs_ids,
        'attention_mask': multiple_choice_input_mask,
        'token_type_ids': multiple_choice_token_type_ids,
    }
    __a    :	List[Any]									=						model(_A			)
    self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.num_choices)			)



   def    __magic_name__    ( self			)    ->   List[str]:
    __a    :	Optional[int]									=						self.prepare_config_and_inputs()
    (
        (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    (
        __a
    )			,    
    )    :	Dict									=						config_and_inputs
    __a    :	List[Any]									=						{
        'input_ids': input_ids,
        'token_type_ids': token_type_ids,
        'langs': token_type_ids,
        'lengths': input_lengths,
    }
    return config, inputs_dict




@require_tf
class 							lowerCAmelCase					(							lowerCamelCase__							,      lowerCamelCase__							,      unittest.TestCase   ):







   """simple docstring"""







   _A						=   (
       (
           TFXLMModel,
           TFXLMWithLMHeadModel,
           TFXLMForSequenceClassification,
           TFXLMForQuestionAnsweringSimple,
           TFXLMForTokenClassification,
           TFXLMForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   _A						=   (
       (TFXLMWithLMHeadModel,) if is_tf_available() else ()
   )  # TODO (PVP): Check other models whether language generation is also applicable
   _A						=   (
       {
           'feature-extraction': TFXLMModel,
           'fill-mask': TFXLMWithLMHeadModel,
           'question-answering': TFXLMForQuestionAnsweringSimple,
           'text-classification': TFXLMForSequenceClassification,
           'text-generation': TFXLMWithLMHeadModel,
           'token-classification': TFXLMForTokenClassification,
           'zero-shot': TFXLMForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   _A						=   False
   _A						=   False



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   _A			)    ->   Union[str, Any]:
    if (
        pipeline_test_casse_name == "QAPipelineTests"
        and tokenizer_name is not None
        and not tokenizer_name.endswith('Fast'			)
    ):
     # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
     # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
     # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
     return True

    return False



   def    __magic_name__    ( self			)    ->   List[Any]:
    __a    :	Tuple									=						TFXLMModelTester(self			)
    __a    :	List[str]									=						ConfigTester(self   ,   config_class=_A   ,   emb_dim=37			)



   def    __magic_name__    ( self			)    ->   List[str]:
    self.config_tester.run_common_tests()



   def    __magic_name__    ( self			)    ->   List[str]:
    __a    :	List[Any]									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_model(*_A			)



   def    __magic_name__    ( self			)    ->   Dict:
    __a    :	Optional[int]									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_lm_head(*_A			)



   def    __magic_name__    ( self			)    ->   Tuple:
    __a    :	Optional[int]									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_qa(*_A			)



   def    __magic_name__    ( self			)    ->   List[Any]:
    __a    :	str									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_sequence_classif(*_A			)



   def    __magic_name__    ( self			)    ->   List[Any]:
    __a    :	str									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_for_token_classification(*_A			)



   def    __magic_name__    ( self			)    ->   Union[str, Any]:
    __a    :	Optional[Any]									=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_xlm_for_multiple_choice(*_A			)



   @slow
   def    __magic_name__    ( self			)    ->   Tuple:
    for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __a    :	int									=						TFXLMModel.from_pretrained(_A			)
     self.assertIsNotNone(_A			)




@require_tf
class 							lowerCAmelCase					(							unittest.TestCase   ):







   """simple docstring"""







   @slow
   def    __magic_name__    ( self			)    ->   Any:
    __a    :	Union[str, Any]									=						TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'			)
    __a    :	int									=						tf.convert_to_tensor([[14, 447]]   ,   dtype=tf.intaa			)  # the president
    __a    :	Optional[int]									=						[
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
        14,
        447,
    ]  # the president the president the president the president the president the president the president the president the president the president
    # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
    __a    :	str									=						model.generate(_A   ,   do_sample=_A			)
    self.assertListEqual(output_ids[0].numpy().tolist()   ,   _A			)












'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import TvltImageProcessor






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__=10       ,		SCREAMING_SNAKE_CASE__=10       ,		SCREAMING_SNAKE_CASE__=False       ,		SCREAMING_SNAKE_CASE__=False		):
 __a    :	Dict									=						[]
 for i in range(image_processor_tester.num_frames		):
  video.append(np.random.randint(255       ,		size=(image_processor_tester.num_channels, width, height)       ,		dtype=np.uinta		)		)

 if not numpify and not torchify:
  # PIL expects the channel dimension as last dimension
  __a    :	List[Any]									=						[Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE__       ,		0       ,		-1		)		) for frame in video]

 if torchify:
  __a    :	List[Any]									=						[torch.from_numpy(SCREAMING_SNAKE_CASE__		) for frame in video]

 return video






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__=False       ,		SCREAMING_SNAKE_CASE__=False       ,		SCREAMING_SNAKE_CASE__=False		):
 assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

 __a    :	Optional[int]									=						[]
 for i in range(image_processor_tester.batch_size		):
  if equal_resolution:
   __a    :	Dict									=						image_processor_tester.max_resolution
  else:
   __a			,    __a    :	Union[str, Any]									=						np.random.choice(
       np.arange(image_processor_tester.min_resolution       ,		image_processor_tester.max_resolution		)       ,		2		)
   __a    :	Any									=						prepare_video(
       image_processor_tester=SCREAMING_SNAKE_CASE__       ,		width=SCREAMING_SNAKE_CASE__       ,		height=SCREAMING_SNAKE_CASE__       ,		numpify=SCREAMING_SNAKE_CASE__       ,		torchify=SCREAMING_SNAKE_CASE__       ,		)
  video_inputs.append(SCREAMING_SNAKE_CASE__		)

 return video_inputs




class 							lowerCAmelCase					(							unittest.TestCase   ):







   """simple docstring"""







   def __init__( self   ,   _A   ,   _A=7   ,   _A=3   ,   _A=4   ,   _A=18   ,   _A=30   ,   _A=400   ,   _A=True   ,   _A=None   ,   _A=True   ,   _A=[0.5, 0.5, 0.5]   ,   _A=[0.5, 0.5, 0.5]   ,   _A=True   ,   _A=None   ,   )    ->   str:
    __a    :	Any									=						size if size is not None else {'shortest_edge': 18}
    __a    :	str									=						crop_size if crop_size is not None else {'height': 18, 'width': 18}

    __a    :	Any									=						parent
    __a    :	Tuple									=						batch_size
    __a    :	Any									=						num_channels
    __a    :	Union[str, Any]									=						num_frames
    __a    :	Optional[Any]									=						image_size
    __a    :	Any									=						min_resolution
    __a    :	List[str]									=						max_resolution
    __a    :	Any									=						do_resize
    __a    :	Dict									=						size
    __a    :	Optional[int]									=						do_normalize
    __a    :	Union[str, Any]									=						image_mean
    __a    :	List[Any]									=						image_std
    __a    :	Dict									=						do_center_crop
    __a    :	List[Any]									=						crop_size



   def    __magic_name__    ( self			)    ->   Any:
    return {
        "image_mean": self.image_mean,
        "image_std": self.image_std,
        "do_normalize": self.do_normalize,
        "do_resize": self.do_resize,
        "size": self.size,
        "do_center_crop": self.do_center_crop,
        "crop_size": self.crop_size,
    }




@require_torch
@require_vision
class 							lowerCAmelCase					(							lowerCamelCase__							,      unittest.TestCase   ):







   """simple docstring"""







   _A						=   TvltImageProcessor if is_vision_available() else None



   def    __magic_name__    ( self			)    ->   int:
    __a    :	List[Any]									=						TvltImageProcessorTester(self			)



   @property
   def    __magic_name__    ( self			)    ->   Optional[Any]:
    return self.image_processor_tester.prepare_image_processor_dict()



   def    __magic_name__    ( self			)    ->   Optional[int]:
    __a    :	Any									=						self.image_processing_class(**self.image_processor_dict			)
    self.assertTrue(hasattr(_A   ,   'image_mean'			)			)
    self.assertTrue(hasattr(_A   ,   'image_std'			)			)
    self.assertTrue(hasattr(_A   ,   'do_normalize'			)			)
    self.assertTrue(hasattr(_A   ,   'do_resize'			)			)
    self.assertTrue(hasattr(_A   ,   'do_center_crop'			)			)
    self.assertTrue(hasattr(_A   ,   'size'			)			)



   def    __magic_name__    ( self			)    ->   str:
    # Initialize image_processor
    __a    :	str									=						self.image_processing_class(**self.image_processor_dict			)
    # create random PIL videos
    __a    :	Optional[Any]									=						prepare_video_inputs(self.image_processor_tester   ,   equal_resolution=_A			)
    for video in video_inputs:
     self.assertIsInstance(_A   ,   _A			)
     self.assertIsInstance(video[0]   ,   Image.Image			)

    # Test not batched input
    __a    :	Tuple									=						image_processor(video_inputs[0]   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            1,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )

    # Test batched
    __a    :	Dict									=						image_processor(_A   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )



   def    __magic_name__    ( self			)    ->   Any:
    # Initialize image_processor
    __a    :	Union[str, Any]									=						self.image_processing_class(**self.image_processor_dict			)
    # create random numpy tensors
    __a    :	List[str]									=						prepare_video_inputs(self.image_processor_tester   ,   equal_resolution=_A   ,   numpify=_A			)
    for video in video_inputs:
     self.assertIsInstance(_A   ,   _A			)
     self.assertIsInstance(video[0]   ,   np.ndarray			)

    # Test not batched input
    __a    :	List[str]									=						image_processor(video_inputs[0]   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            1,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )

    # Test batched
    __a    :	Dict									=						image_processor(_A   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )



   def    __magic_name__    ( self			)    ->   Union[str, Any]:
    # Initialize image_processor
    __a    :	Optional[Any]									=						self.image_processing_class(**self.image_processor_dict			)
    # create random PyTorch tensors
    __a    :	Optional[Any]									=						prepare_video_inputs(self.image_processor_tester   ,   equal_resolution=_A   ,   torchify=_A			)
    for video in video_inputs:
     self.assertIsInstance(_A   ,   _A			)
     self.assertIsInstance(video[0]   ,   torch.Tensor			)

    # Test not batched input
    __a    :	Dict									=						image_processor(video_inputs[0]   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            1,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )

    # Test batched
    __a    :	Tuple									=						image_processor(_A   ,   return_tensors='pt'			).pixel_values
    self.assertEqual(
        encoded_videos.shape   ,   (
            self.image_processor_tester.batch_size,
            self.image_processor_tester.num_frames,
            self.image_processor_tester.num_channels,
            self.image_processor_tester.crop_size['height'],
            self.image_processor_tester.crop_size['width'],
        )   ,   )




