




'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
							    TFXLMForMultipleChoice,
							    TFXLMForQuestionAnsweringSimple,
							    TFXLMForSequenceClassification,
							    TFXLMForTokenClassification,
							    TFXLMModel,
							    TFXLMWithLMHeadModel,
							    XLMConfig,
							)
class      __UpperCAmelCase							:



					'''simple docstring'''





					def __init__(  self    :	List[Any]			,					_lowercase    :	Any			,					)						->       str:
								A_       		=   parent
								A_       		=   13
								A_       		=   7
								A_       		=   True
								A_       		=   True
								A_       		=   True
								A_       		=   True
								A_       		=   True
								A_       		=   False
								A_       		=   False
								A_       		=   False
								A_       		=   2
								A_       		=   99
								A_       		=   0
								A_       		=   32
								A_       		=   2
								A_       		=   4
								A_       		=   0.1
								A_       		=   0.1
								A_       		=   512
								A_       		=   16
								A_       		=   2
								A_       		=   0.02
								A_       		=   3
								A_       		=   4
								A_       		=   'last'
								A_       		=   True
								A_       		=   None
								A_       		=   0



					def 						__snake_case	(  self    :	List[str])						->       Optional[int]:
								A_       		=   ids_tensor([self.batch_size, self.seq_length]			,					self.vocab_size)
								A_       		=   random_attention_mask([self.batch_size, self.seq_length]			,					dtype=tf.floataa)

								A_       		=   None
								if self.use_input_lengths:
											A_       		=   (
											    ids_tensor([self.batch_size]			,					vocab_size=2) + self.seq_length - 2
											)  # small variation of seq_length

								A_       		=   None
								if self.use_token_type_ids:
											A_       		=   ids_tensor([self.batch_size, self.seq_length]			,					self.n_langs)

								A_       		=   None
								A_       		=   None
								A_       		=   None
								if self.use_labels:
											A_       		=   ids_tensor([self.batch_size]			,					self.type_sequence_label_size)
											A_       		=   ids_tensor([self.batch_size, self.seq_length]			,					self.num_labels)
											A_       		=   ids_tensor([self.batch_size]			,					2			,					dtype=tf.floataa)
											A_       		=   ids_tensor([self.batch_size]			,					self.num_choices)

								A_       		=   XLMConfig(
								    vocab_size=self.vocab_size			,					n_special=self.n_special			,					emb_dim=self.hidden_size			,					n_layers=self.num_hidden_layers			,					n_heads=self.num_attention_heads			,					dropout=self.hidden_dropout_prob			,					attention_dropout=self.attention_probs_dropout_prob			,					gelu_activation=self.gelu_activation			,					sinusoidal_embeddings=self.sinusoidal_embeddings			,					asm=self.asm			,					causal=self.causal			,					n_langs=self.n_langs			,					max_position_embeddings=self.max_position_embeddings			,					initializer_range=self.initializer_range			,					summary_type=self.summary_type			,					use_proj=self.use_proj			,					bos_token_id=self.bos_token_id			,					)

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_lengths,
								    sequence_labels,
								    token_labels,
								    is_impossible_labels,
								    choice_labels,
								    input_mask,
								)



					def 						__snake_case	(  self    :	Any			,					_lowercase    :	List[str]			,					_lowercase    :	Optional[Any]			,					_lowercase    :	List[Any]			,					_lowercase    :	int			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Tuple			,					_lowercase    :	str			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Optional[int]			,					)						->       int:
								A_       		=   TFXLMModel(config=_lowercase)
								A_       		=   {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
								A_       		=   model(_lowercase)

								A_       		=   [input_ids, input_mask]
								A_       		=   model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size))



					def 						__snake_case	(  self    :	str			,					_lowercase    :	Any			,					_lowercase    :	Any			,					_lowercase    :	List[Any]			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	str			,					_lowercase    :	List[Any]			,					_lowercase    :	Optional[Any]			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Tuple			,					)						->       str:
								A_       		=   TFXLMWithLMHeadModel(_lowercase)

								A_       		=   {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
								A_       		=   model(_lowercase)

								A_       		=   outputs

								self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.vocab_size))



					def 						__snake_case	(  self    :	Union[str, Any]			,					_lowercase    :	int			,					_lowercase    :	Tuple			,					_lowercase    :	Optional[int]			,					_lowercase    :	List[str]			,					_lowercase    :	Optional[Any]			,					_lowercase    :	List[str]			,					_lowercase    :	Optional[Any]			,					_lowercase    :	Optional[int]			,					_lowercase    :	Optional[Any]			,					)						->       List[str]:
								A_       		=   TFXLMForQuestionAnsweringSimple(_lowercase)

								A_       		=   {'input_ids': input_ids, 'lengths': input_lengths}

								A_       		=   model(_lowercase)

								self.parent.assertEqual(result.start_logits.shape			,					(self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape			,					(self.batch_size, self.seq_length))



					def 						__snake_case	(  self    :	Optional[Any]			,					_lowercase    :	List[Any]			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Optional[Any]			,					_lowercase    :	List[Any]			,					_lowercase    :	Any			,					_lowercase    :	Optional[Any]			,					_lowercase    :	Optional[int]			,					_lowercase    :	List[str]			,					_lowercase    :	int			,					)						->       List[str]:
								A_       		=   TFXLMForSequenceClassification(_lowercase)

								A_       		=   {'input_ids': input_ids, 'lengths': input_lengths}

								A_       		=   model(_lowercase)

								self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.type_sequence_label_size))



					def 						__snake_case	(  self    :	List[str]			,					_lowercase    :	Tuple			,					_lowercase    :	Any			,					_lowercase    :	int			,					_lowercase    :	str			,					_lowercase    :	Tuple			,					_lowercase    :	Any			,					_lowercase    :	Dict			,					_lowercase    :	int			,					_lowercase    :	Tuple			,					)						->       int:
								A_       		=   self.num_labels
								A_       		=   TFXLMForTokenClassification(config=_lowercase)
								A_       		=   {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_       		=   model(_lowercase)
								self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.num_labels))



					def 						__snake_case	(  self    :	int			,					_lowercase    :	int			,					_lowercase    :	str			,					_lowercase    :	List[str]			,					_lowercase    :	Tuple			,					_lowercase    :	Optional[Any]			,					_lowercase    :	Any			,					_lowercase    :	str			,					_lowercase    :	Any			,					_lowercase    :	List[Any]			,					)						->       Optional[Any]:
								A_       		=   self.num_choices
								A_       		=   TFXLMForMultipleChoice(config=_lowercase)
								A_       		=   tf.tile(tf.expand_dims(_lowercase			,					1)			,					(1, self.num_choices, 1))
								A_       		=   tf.tile(tf.expand_dims(_lowercase			,					1)			,					(1, self.num_choices, 1))
								A_       		=   tf.tile(tf.expand_dims(_lowercase			,					1)			,					(1, self.num_choices, 1))
								A_       		=   {
								    'input_ids': multiple_choice_inputs_ids,
								    'attention_mask': multiple_choice_input_mask,
								    'token_type_ids': multiple_choice_token_type_ids,
								}
								A_       		=   model(_lowercase)
								self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.num_choices))



					def 						__snake_case	(  self    :	Dict)						->       List[str]:
								A_       		=   self.prepare_config_and_inputs()
								(
								    (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   (
								    A_
								)		,   
								)       		=   config_and_inputs
								A_       		=   {
								    'input_ids': input_ids,
								    'token_type_ids': token_type_ids,
								    'langs': token_type_ids,
								    'lengths': input_lengths,
								}
								return config, inputs_dict





@require_tf
class      __UpperCAmelCase							(    lowerCAmelCase ,lowerCAmelCase ,unittest.TestCase							):



					'''simple docstring'''





					_UpperCamelCase  	=		(
					    (
					        TFXLMModel,
					        TFXLMWithLMHeadModel,
					        TFXLMForSequenceClassification,
					        TFXLMForQuestionAnsweringSimple,
					        TFXLMForTokenClassification,
					        TFXLMForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					_UpperCamelCase  	=		(
					    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
					)  # TODO (PVP): Check other models whether language generation is also applicable
					_UpperCamelCase  	=		(
					    {
					        """feature-extraction""": TFXLMModel,
					        """fill-mask""": TFXLMWithLMHeadModel,
					        """question-answering""": TFXLMForQuestionAnsweringSimple,
					        """text-classification""": TFXLMForSequenceClassification,
					        """text-generation""": TFXLMWithLMHeadModel,
					        """token-classification""": TFXLMForTokenClassification,
					        """zero-shot""": TFXLMForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					_UpperCamelCase  	=		False
					_UpperCamelCase  	=		False



					def 						__snake_case	(  self    :	str			,					_lowercase    :	Dict			,					_lowercase    :	str			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Union[str, Any]			,					_lowercase    :	Tuple)						->       List[Any]:
								if (
								    pipeline_test_casse_name == "QAPipelineTests"
								    and tokenizer_name is not None
								    and not tokenizer_name.endswith('Fast')
								):
											# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
											# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
											# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
											return True

								return False



					def 						__snake_case	(  self    :	Any)						->       Optional[Any]:
								A_       		=   TFXLMModelTester(self)
								A_       		=   ConfigTester(self			,					config_class=_lowercase			,					emb_dim=37)



					def 						__snake_case	(  self    :	int)						->       Dict:
								self.config_tester.run_common_tests()



					def 						__snake_case	(  self    :	Any)						->       Union[str, Any]:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_model(*_lowercase)



					def 						__snake_case	(  self    :	Any)						->       Dict:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_lm_head(*_lowercase)



					def 						__snake_case	(  self    :	str)						->       List[Any]:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_qa(*_lowercase)



					def 						__snake_case	(  self    :	List[Any])						->       Union[str, Any]:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_sequence_classif(*_lowercase)



					def 						__snake_case	(  self    :	Union[str, Any])						->       Any:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_for_token_classification(*_lowercase)



					def 						__snake_case	(  self    :	str)						->       List[Any]:
								A_       		=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_xlm_for_multiple_choice(*_lowercase)



					@slow
					def 						__snake_case	(  self    :	Union[str, Any])						->       Optional[Any]:
								for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A_       		=   TFXLMModel.from_pretrained(_lowercase)
											self.assertIsNotNone(_lowercase)





@require_tf
class      __UpperCAmelCase							(    unittest.TestCase							):



					'''simple docstring'''





					@slow
					def 						__snake_case	(  self    :	Tuple)						->       Any:
								A_       		=   TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048')
								A_       		=   tf.convert_to_tensor([[14, 447]]			,					dtype=tf.intaa)  # the president
								A_       		=   [
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								    14,
								    447,
								]  # the president the president the president the president the president the president the president the president the president the president
								# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
								A_       		=   model.generate(_lowercase			,					do_sample=_lowercase)
								self.assertListEqual(output_ids[0].numpy().tolist()			,					_lowercase)










'''simple docstring'''
from unittest.mock import patch

import datasets
from datasets import Dataset





def  lowerCamelCase(       )		->		Any:
			A_       		=   Dataset.from_dict({'col_1': [3, 2, 0, 1]}    )

			with patch('tqdm.auto.tqdm'    ) as mock_tqdm:
						datasets.disable_progress_bar()
						dset.map(lambda SCREAMING_SNAKE_CASE_    : {"col_2": x["col_1"] + 1}    )
						mock_tqdm.assert_not_called()

						mock_tqdm.reset_mock()

						datasets.enable_progress_bar()
						dset.map(lambda SCREAMING_SNAKE_CASE_    : {"col_2": x["col_1"] + 1}    )
						mock_tqdm.assert_called()




