






import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCAmelCase__			 =	logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCAmelCase__			 =	direct_transformers_import(Path(__file__).parent)


UpperCAmelCase__			 =	{
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}




class        snake_case_					(						__UpperCamelCase			):

						"""simple docstring"""



						snake_case__   					=			["""feature_extractor""", """tokenizer"""]
						# Names need to be attr_class for attr in attributes
						snake_case__   					=			None
						snake_case__   					=			None
						snake_case__   					=			None



						def __init__(self:				Optional[int]   ,				*__UpperCAmelCase:				Optional[Any]   ,				**__UpperCAmelCase:				List[Any]      )						->    Optional[int]:




											'''simple docstring'''



											for key in kwargs:
																if key not in self.attributes:
																					raise TypeError(f'Unexpected keyword argument {key}.'      )
											for arg, attribute_name in zip(__UpperCAmelCase   ,				self.attributes      ):
																if attribute_name in kwargs:
																					raise TypeError(f'Got multiple values for argument {attribute_name}.'      )
																else:
																					__a				:	Tuple			=				arg

											if len(__UpperCAmelCase      ) != len(self.attributes      ):
																raise ValueError(
																    f'This processor requires {len(self.attributes      )} arguments: {", ".join(self.attributes      )}. Got '
																    f'{len(__UpperCAmelCase      )} arguments instead.'      )

											# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
											for attribute_name, arg in kwargs.items():
																__a				:	Optional[int]			=				getattr(self   ,				f'{attribute_name}_class'      )
																# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
																__a				:	Dict			=				AUTO_TO_BASE_CLASS_MAPPING.get(__UpperCAmelCase   ,				__UpperCAmelCase      )
																if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																					__a				:	Union[str, Any]			=				tuple(getattr(__UpperCAmelCase   ,				__UpperCAmelCase      ) for n in class_name if n is not None      )
																else:
																					__a				:	int			=				getattr(__UpperCAmelCase   ,				__UpperCAmelCase      )

																if not isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																					raise ValueError(
																					    f'Received a {type(__UpperCAmelCase      ).__name__} for argument {attribute_name}, but a {class_name} was expected.'      )

																setattr(self   ,				__UpperCAmelCase   ,				__UpperCAmelCase      )



						def __repr__(self:				List[str]      )						->    Union[str, Any]:




											'''simple docstring'''



											__a				:	Optional[Any]			=				[f'- {name}: {repr(getattr(self   ,				__UpperCAmelCase      )      )}' for name in self.attributes]
											__a				:	Dict			=				"\n".join(__UpperCAmelCase      )
											return f'{self.__class__.__name__}:\n{attributes_repr}'



						def 					UpperCAmelCase__				(self:				List[str]   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				bool = False   ,				**__UpperCAmelCase:				int      )						->    Union[str, Any]:




											'''simple docstring'''



											os.makedirs(__UpperCAmelCase   ,				exist_ok=__UpperCAmelCase      )

											if push_to_hub:
																__a				:	Union[str, Any]			=				kwargs.pop("commit_message"   ,				__UpperCAmelCase      )
																__a				:	int			=				kwargs.pop("repo_id"   ,				save_directory.split(os.path.sep      )[-1]      )
																__a				:	List[str]			=				self._create_repo(__UpperCAmelCase   ,				**__UpperCAmelCase      )
																__a				:	Optional[int]			=				self._get_files_timestamps(__UpperCAmelCase      )
											# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
											# loaded from the Hub.
											if self._auto_class is not None:
																__a				:	List[Any]			=				[getattr(self   ,				__UpperCAmelCase      ) for attribute_name in self.attributes]
																__a				:	Any			=				[(a.init_kwargs if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ) else a) for a in attrs]
																custom_object_save(self   ,				__UpperCAmelCase   ,				config=__UpperCAmelCase      )

											for attribute_name in self.attributes:
																__a				:	Any			=				getattr(self   ,				__UpperCAmelCase      )
																# Include the processor class in the attribute config so this processor can then be reloaded with the
																# `AutoProcessor` API.
																if hasattr(__UpperCAmelCase   ,				"_set_processor_class"      ):
																					attribute._set_processor_class(self.__class__.__name__      )
																attribute.save_pretrained(__UpperCAmelCase      )

											if self._auto_class is not None:
																# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
																for attribute_name in self.attributes:
																					__a				:	int			=				getattr(self   ,				__UpperCAmelCase      )
																					if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																										del attribute.init_kwargs["auto_map"]

											if push_to_hub:
																self._upload_modified_files(
																    __UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase   ,				commit_message=__UpperCAmelCase   ,				token=kwargs.get("use_auth_token"      )   ,				)



						@classmethod
						def 					UpperCAmelCase__				(cls:				Dict   ,				__UpperCAmelCase:				Union[str, os.PathLike]   ,				__UpperCAmelCase:				Optional[Union[str, os.PathLike]] = None   ,				__UpperCAmelCase:				bool = False   ,				__UpperCAmelCase:				bool = False   ,				__UpperCAmelCase:				Optional[Union[str, bool]] = None   ,				__UpperCAmelCase:				str = "main"   ,				**__UpperCAmelCase:				str   ,				)						->    Optional[int]:




											'''simple docstring'''



											__a				:	Any			=				cache_dir
											__a				:	Optional[Any]			=				force_download
											__a				:	Optional[int]			=				local_files_only
											__a				:	List[str]			=				revision

											__a				:	Any			=				kwargs.pop("use_auth_token"   ,				__UpperCAmelCase      )
											if use_auth_token is not None:
																warnings.warn(
																    "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."   ,				__UpperCAmelCase      )
																if token is not None:
																					raise ValueError(
																					    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."      )
																__a				:	Any			=				use_auth_token

											if token is not None:
																# change to `token` in a follow-up PR
																__a				:	Optional[int]			=				token

											__a				:	Tuple			=				cls._get_arguments_from_pretrained(__UpperCAmelCase   ,				**__UpperCAmelCase      )
											return cls(*__UpperCAmelCase      )



						@classmethod
						def 					UpperCAmelCase__				(cls:				int   ,				__UpperCAmelCase:				Dict="AutoProcessor"      )						->    List[str]:




											'''simple docstring'''



											if not isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																__a				:	str			=				auto_class.__name__

											import transformers.models.auto as auto_module

											if not hasattr(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																raise ValueError(f'{auto_class} is not a valid auto class.'      )

											__a				:	Any			=				auto_class



						@classmethod
						def 					UpperCAmelCase__				(cls:				Optional[Any]   ,				__UpperCAmelCase:				Union[str, Any]   ,				**__UpperCAmelCase:				Tuple      )						->    Any:




											'''simple docstring'''



											__a				:	Optional[Any]			=				[]
											for attribute_name in cls.attributes:
																__a				:	Dict			=				getattr(cls   ,				f'{attribute_name}_class'      )
																if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																					__a				:	str			=				tuple(getattr(__UpperCAmelCase   ,				__UpperCAmelCase      ) if n is not None else None for n in class_name      )
																					__a				:	Any			=				kwargs.get("use_fast"   ,				__UpperCAmelCase      )
																					if use_fast and classes[1] is not None:
																										__a				:	List[Any]			=				classes[1]
																					else:
																										__a				:	Any			=				classes[0]
																else:
																					__a				:	str			=				getattr(__UpperCAmelCase   ,				__UpperCAmelCase      )

																args.append(attribute_class.from_pretrained(__UpperCAmelCase   ,				**__UpperCAmelCase      )      )
											return args



						@property
						def 					UpperCAmelCase__				(self:				Any      )						->    str:




											'''simple docstring'''



											__a				:	Optional[int]			=				getattr(self   ,				self.attributes[0]      )
											return getattr(__UpperCAmelCase   ,				"model_input_names"   ,				__UpperCAmelCase      )


UpperCAmelCase__			 =	copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
			UpperCAmelCase__			 =	ProcessorMixin.push_to_hub.__doc__.format(
			    object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
			)













from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
			    TFXLMForMultipleChoice,
			    TFXLMForQuestionAnsweringSimple,
			    TFXLMForSequenceClassification,
			    TFXLMForTokenClassification,
			    TFXLMModel,
			    TFXLMWithLMHeadModel,
			    XLMConfig,
			)




class        snake_case_					:

						"""simple docstring"""



						def __init__(self:				List[Any]   ,				__UpperCAmelCase:				str   ,				)						->    int:




											'''simple docstring'''



											__a				:	str			=				parent
											__a				:	Dict			=				13
											__a				:	Tuple			=				7
											__a				:	Union[str, Any]			=				True
											__a				:	Dict			=				True
											__a				:	Optional[Any]			=				True
											__a				:	Tuple			=				True
											__a				:	int			=				True
											__a				:	Tuple			=				False
											__a				:	Optional[int]			=				False
											__a				:	str			=				False
											__a				:	Optional[Any]			=				2
											__a				:	Optional[Any]			=				99
											__a				:	List[str]			=				0
											__a				:	int			=				32
											__a				:	List[Any]			=				2
											__a				:	Union[str, Any]			=				4
											__a				:	List[str]			=				0.1
											__a				:	Tuple			=				0.1
											__a				:	str			=				512
											__a				:	Any			=				16
											__a				:	Optional[Any]			=				2
											__a				:	str			=				0.02
											__a				:	Union[str, Any]			=				3
											__a				:	Union[str, Any]			=				4
											__a				:	Optional[int]			=				"last"
											__a				:	List[str]			=				True
											__a				:	List[Any]			=				None
											__a				:	Any			=				0



						def 					UpperCAmelCase__				(self:				int      )						->    str:




											'''simple docstring'''



											__a				:	Optional[Any]			=				ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size      )
											__a				:	Union[str, Any]			=				random_attention_mask([self.batch_size, self.seq_length]   ,				dtype=tf.floataa      )

											__a				:	Dict			=				None
											if self.use_input_lengths:
																__a				:	List[Any]			=				(
																    ids_tensor([self.batch_size]   ,				vocab_size=2      ) + self.seq_length - 2
																)  # small variation of seq_length

											__a				:	int			=				None
											if self.use_token_type_ids:
																__a				:	Tuple			=				ids_tensor([self.batch_size, self.seq_length]   ,				self.n_langs      )

											__a				:	Union[str, Any]			=				None
											__a				:	str			=				None
											__a				:	str			=				None
											if self.use_labels:
																__a				:	Dict			=				ids_tensor([self.batch_size]   ,				self.type_sequence_label_size      )
																__a				:	List[Any]			=				ids_tensor([self.batch_size, self.seq_length]   ,				self.num_labels      )
																__a				:	int			=				ids_tensor([self.batch_size]   ,				2   ,				dtype=tf.floataa      )
																__a				:	Tuple			=				ids_tensor([self.batch_size]   ,				self.num_choices      )

											__a				:	str			=				XLMConfig(
											    vocab_size=self.vocab_size   ,				n_special=self.n_special   ,				emb_dim=self.hidden_size   ,				n_layers=self.num_hidden_layers   ,				n_heads=self.num_attention_heads   ,				dropout=self.hidden_dropout_prob   ,				attention_dropout=self.attention_probs_dropout_prob   ,				gelu_activation=self.gelu_activation   ,				sinusoidal_embeddings=self.sinusoidal_embeddings   ,				asm=self.asm   ,				causal=self.causal   ,				n_langs=self.n_langs   ,				max_position_embeddings=self.max_position_embeddings   ,				initializer_range=self.initializer_range   ,				summary_type=self.summary_type   ,				use_proj=self.use_proj   ,				bos_token_id=self.bos_token_id   ,				)

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_lengths,
											    sequence_labels,
											    token_labels,
											    is_impossible_labels,
											    choice_labels,
											    input_mask,
											)



						def 					UpperCAmelCase__				(self:				int   ,				__UpperCAmelCase:				Union[str, Any]   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				int   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Any   ,				)						->    List[Any]:




											'''simple docstring'''



											__a				:	Union[str, Any]			=				TFXLMModel(config=__UpperCAmelCase      )
											__a				:	Tuple			=				{"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
											__a				:	List[str]			=				model(__UpperCAmelCase      )

											__a				:	List[Any]			=				[input_ids, input_mask]
											__a				:	str			=				model(__UpperCAmelCase      )
											self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size)      )



						def 					UpperCAmelCase__				(self:				Tuple   ,				__UpperCAmelCase:				Optional[Any]   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				int   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				Optional[Any]   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				List[str]   ,				)						->    Union[str, Any]:




											'''simple docstring'''



											__a				:	int			=				TFXLMWithLMHeadModel(__UpperCAmelCase      )

											__a				:	List[Any]			=				{"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
											__a				:	int			=				model(__UpperCAmelCase      )

											__a				:	int			=				outputs

											self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)      )



						def 					UpperCAmelCase__				(self:				Optional[int]   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Optional[Any]   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Union[str, Any]   ,				__UpperCAmelCase:				Optional[Any]   ,				__UpperCAmelCase:				List[Any]   ,				)						->    List[str]:




											'''simple docstring'''



											__a				:	Tuple			=				TFXLMForQuestionAnsweringSimple(__UpperCAmelCase      )

											__a				:	Union[str, Any]			=				{"input_ids": input_ids, "lengths": input_lengths}

											__a				:	int			=				model(__UpperCAmelCase      )

											self.parent.assertEqual(result.start_logits.shape   ,				(self.batch_size, self.seq_length)      )
											self.parent.assertEqual(result.end_logits.shape   ,				(self.batch_size, self.seq_length)      )



						def 					UpperCAmelCase__				(self:				List[Any]   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				str   ,				__UpperCAmelCase:				int   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				Dict   ,				)						->    str:




											'''simple docstring'''



											__a				:	Dict			=				TFXLMForSequenceClassification(__UpperCAmelCase      )

											__a				:	Union[str, Any]			=				{"input_ids": input_ids, "lengths": input_lengths}

											__a				:	Any			=				model(__UpperCAmelCase      )

											self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.type_sequence_label_size)      )



						def 					UpperCAmelCase__				(self:				Dict   ,				__UpperCAmelCase:				List[str]   ,				__UpperCAmelCase:				Optional[Any]   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				str   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				int   ,				)						->    int:




											'''simple docstring'''



											__a				:	Optional[int]			=				self.num_labels
											__a				:	Any			=				TFXLMForTokenClassification(config=__UpperCAmelCase      )
											__a				:	Dict			=				{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
											__a				:	List[Any]			=				model(__UpperCAmelCase      )
											self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.num_labels)      )



						def 					UpperCAmelCase__				(self:				Optional[int]   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Optional[int]   ,				__UpperCAmelCase:				Optional[int]   ,				)						->    List[str]:




											'''simple docstring'''



											__a				:	List[str]			=				self.num_choices
											__a				:	str			=				TFXLMForMultipleChoice(config=__UpperCAmelCase      )
											__a				:	List[str]			=				tf.tile(tf.expand_dims(__UpperCAmelCase   ,				1      )   ,				(1, self.num_choices, 1)      )
											__a				:	Tuple			=				tf.tile(tf.expand_dims(__UpperCAmelCase   ,				1      )   ,				(1, self.num_choices, 1)      )
											__a				:	int			=				tf.tile(tf.expand_dims(__UpperCAmelCase   ,				1      )   ,				(1, self.num_choices, 1)      )
											__a				:	Dict			=				{
											    "input_ids": multiple_choice_inputs_ids,
											    "attention_mask": multiple_choice_input_mask,
											    "token_type_ids": multiple_choice_token_type_ids,
											}
											__a				:	Optional[Any]			=				model(__UpperCAmelCase      )
											self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_choices)      )



						def 					UpperCAmelCase__				(self:				Tuple      )						->    Optional[Any]:




											'''simple docstring'''



											__a				:	Tuple			=				self.prepare_config_and_inputs()
											(
											    (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, (
											    __a
											)			, 
											)				:	List[Any]			=				config_and_inputs
											__a				:	Optional[Any]			=				{
											    "input_ids": input_ids,
											    "token_type_ids": token_type_ids,
											    "langs": token_type_ids,
											    "lengths": input_lengths,
											}
											return config, inputs_dict




@require_tf
class        snake_case_					(						__UpperCamelCase		,       __UpperCamelCase		,       unittest.TestCase			):

						"""simple docstring"""



						snake_case__   					=			(
						    (
						        TFXLMModel,
						        TFXLMWithLMHeadModel,
						        TFXLMForSequenceClassification,
						        TFXLMForQuestionAnsweringSimple,
						        TFXLMForTokenClassification,
						        TFXLMForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						snake_case__   					=			(
						    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
						)  # TODO (PVP): Check other models whether language generation is also applicable
						snake_case__   					=			(
						    {
						        """feature-extraction""": TFXLMModel,
						        """fill-mask""": TFXLMWithLMHeadModel,
						        """question-answering""": TFXLMForQuestionAnsweringSimple,
						        """text-classification""": TFXLMForSequenceClassification,
						        """text-generation""": TFXLMWithLMHeadModel,
						        """token-classification""": TFXLMForTokenClassification,
						        """zero-shot""": TFXLMForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						snake_case__   					=			False
						snake_case__   					=			False



						def 					UpperCAmelCase__				(self:				Union[str, Any]   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				List[Any]   ,				__UpperCAmelCase:				Union[str, Any]   ,				__UpperCAmelCase:				Tuple      )						->    Tuple:




											'''simple docstring'''



											if (
											    pipeline_test_casse_name == "QAPipelineTests"
											    and tokenizer_name is not None
											    and not tokenizer_name.endswith("Fast"      )
											):
																# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																return True

											return False



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    int:




											'''simple docstring'''



											__a				:	List[Any]			=				TFXLMModelTester(self      )
											__a				:	int			=				ConfigTester(self   ,				config_class=__UpperCAmelCase   ,				emb_dim=37      )



						def 					UpperCAmelCase__				(self:				Tuple      )						->    Optional[Any]:




											'''simple docstring'''



											self.config_tester.run_common_tests()



						def 					UpperCAmelCase__				(self:				Any      )						->    int:




											'''simple docstring'''



											__a				:	Optional[int]			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_model(*__UpperCAmelCase      )



						def 					UpperCAmelCase__				(self:				Dict      )						->    Dict:




											'''simple docstring'''



											__a				:	str			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_lm_head(*__UpperCAmelCase      )



						def 					UpperCAmelCase__				(self:				Optional[int]      )						->    Optional[int]:




											'''simple docstring'''



											__a				:	List[str]			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_qa(*__UpperCAmelCase      )



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    Any:




											'''simple docstring'''



											__a				:	List[Any]			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_sequence_classif(*__UpperCAmelCase      )



						def 					UpperCAmelCase__				(self:				Optional[int]      )						->    Optional[Any]:




											'''simple docstring'''



											__a				:	Any			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_for_token_classification(*__UpperCAmelCase      )



						def 					UpperCAmelCase__				(self:				Any      )						->    List[Any]:




											'''simple docstring'''



											__a				:	List[str]			=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_xlm_for_multiple_choice(*__UpperCAmelCase      )



						@slow
						def 					UpperCAmelCase__				(self:				List[str]      )						->    Optional[int]:




											'''simple docstring'''



											for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__a				:	Optional[int]			=				TFXLMModel.from_pretrained(__UpperCAmelCase      )
																self.assertIsNotNone(__UpperCAmelCase      )




@require_tf
class        snake_case_					(						unittest.TestCase			):

						"""simple docstring"""



						@slow
						def 					UpperCAmelCase__				(self:				Optional[int]      )						->    str:




											'''simple docstring'''



											__a				:	Any			=				TFXLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"      )
											__a				:	List[Any]			=				tf.convert_to_tensor([[14, 447]]   ,				dtype=tf.intaa      )  # the president
											__a				:	Any			=				[
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											    14,
											    447,
											]  # the president the president the president the president the president the president the president the president the president the president
											# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
											__a				:	Tuple			=				model.generate(__UpperCAmelCase   ,				do_sample=__UpperCAmelCase      )
											self.assertListEqual(output_ids[0].numpy().tolist()   ,				__UpperCAmelCase      )





