





"""simple docstring"""
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )






class       _SCREAMING_SNAKE_CASE:




       def __init__(		self	,SCREAMING_SNAKE_CASE__	,)    ->   List[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :List[str]	       =						parent
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						13
             __SCREAMING_SNAKE_CASE     :Any	       =						7
             __SCREAMING_SNAKE_CASE     :Tuple	       =						True
             __SCREAMING_SNAKE_CASE     :Any	       =						True
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						True
             __SCREAMING_SNAKE_CASE     :Tuple	       =						True
             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						True
             __SCREAMING_SNAKE_CASE     :Tuple	       =						False
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						False
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						False
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						2
             __SCREAMING_SNAKE_CASE     :Any	       =						99
             __SCREAMING_SNAKE_CASE     :str	       =						0
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						32
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						2
             __SCREAMING_SNAKE_CASE     :int	       =						4
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						0.1
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						0.1
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						5_12
             __SCREAMING_SNAKE_CASE     :Tuple	       =						16
             __SCREAMING_SNAKE_CASE     :Tuple	       =						2
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						0.0_2
             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						3
             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						4
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						"""last"""
             __SCREAMING_SNAKE_CASE     :List[str]	       =						True
             __SCREAMING_SNAKE_CASE     :Any	       =						None
             __SCREAMING_SNAKE_CASE     :Any	       =						0




       def 					_UpperCamelCase  (		self							)    ->   int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :int	       =						ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size							)
             __SCREAMING_SNAKE_CASE     :Tuple	       =						random_attention_mask([self.batch_size, self.seq_length]	,dtype=tf.floataa							)

             __SCREAMING_SNAKE_CASE     :Dict	       =						None
             if self.use_input_lengths:
                   __SCREAMING_SNAKE_CASE     :Tuple	       =						(
                       ids_tensor([self.batch_size]	,vocab_size=2							) + self.seq_length - 2
                   )  # small variation of seq_length

             __SCREAMING_SNAKE_CASE     :int	       =						None
             if self.use_token_type_ids:
                   __SCREAMING_SNAKE_CASE     :str	       =						ids_tensor([self.batch_size, self.seq_length]	,self.n_langs							)

             __SCREAMING_SNAKE_CASE     :Tuple	       =						None
             __SCREAMING_SNAKE_CASE     :Dict	       =						None
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						None
             if self.use_labels:
                   __SCREAMING_SNAKE_CASE     :Optional[int]	       =						ids_tensor([self.batch_size]	,self.type_sequence_label_size							)
                   __SCREAMING_SNAKE_CASE     :int	       =						ids_tensor([self.batch_size, self.seq_length]	,self.num_labels							)
                   __SCREAMING_SNAKE_CASE     :Dict	       =						ids_tensor([self.batch_size]	,2	,dtype=tf.floataa							)
                   __SCREAMING_SNAKE_CASE     :List[str]	       =						ids_tensor([self.batch_size]	,self.num_choices							)

             __SCREAMING_SNAKE_CASE     :Tuple	       =						XLMConfig(
                 vocab_size=self.vocab_size	,n_special=self.n_special	,emb_dim=self.hidden_size	,n_layers=self.num_hidden_layers	,n_heads=self.num_attention_heads	,dropout=self.hidden_dropout_prob	,attention_dropout=self.attention_probs_dropout_prob	,gelu_activation=self.gelu_activation	,sinusoidal_embeddings=self.sinusoidal_embeddings	,asm=self.asm	,causal=self.causal	,n_langs=self.n_langs	,max_position_embeddings=self.max_position_embeddings	,initializer_range=self.initializer_range	,summary_type=self.summary_type	,use_proj=self.use_proj	,bos_token_id=self.bos_token_id	,)

             return (
                 config,
                 input_ids,
                 token_type_ids,
                 input_lengths,
                 sequence_labels,
                 token_labels,
                 is_impossible_labels,
                 choice_labels,
                 input_mask,
             )




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						TFXLMModel(config=_lowerCAmelCase							)
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						model(_lowerCAmelCase							)

             __SCREAMING_SNAKE_CASE     :str	       =						[input_ids, input_mask]
             __SCREAMING_SNAKE_CASE     :str	       =						model(_lowerCAmelCase							)
             self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						TFXLMWithLMHeadModel(_lowerCAmelCase							)

             __SCREAMING_SNAKE_CASE     :int	       =						{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
             __SCREAMING_SNAKE_CASE     :List[str]	       =						model(_lowerCAmelCase							)

             __SCREAMING_SNAKE_CASE     :str	       =						outputs

             self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Tuple	       =						TFXLMForQuestionAnsweringSimple(_lowerCAmelCase							)

             __SCREAMING_SNAKE_CASE     :Dict	       =						{"""input_ids""": input_ids, """lengths""": input_lengths}

             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						model(_lowerCAmelCase							)

             self.parent.assertEqual(result.start_logits.shape	,(self.batch_size, self.seq_length)							)
             self.parent.assertEqual(result.end_logits.shape	,(self.batch_size, self.seq_length)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   List[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :List[Any]	       =						TFXLMForSequenceClassification(_lowerCAmelCase							)

             __SCREAMING_SNAKE_CASE     :Tuple	       =						{"""input_ids""": input_ids, """lengths""": input_lengths}

             __SCREAMING_SNAKE_CASE     :Tuple	       =						model(_lowerCAmelCase							)

             self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.type_sequence_label_size)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						self.num_labels
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						TFXLMForTokenClassification(config=_lowerCAmelCase							)
             __SCREAMING_SNAKE_CASE     :str	       =						{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						model(_lowerCAmelCase							)
             self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.num_labels)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,)    ->   Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :List[Any]	       =						self.num_choices
             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						TFXLMForMultipleChoice(config=_lowerCAmelCase							)
             __SCREAMING_SNAKE_CASE     :Tuple	       =						tf.tile(tf.expand_dims(_lowerCAmelCase	,1							)	,(1, self.num_choices, 1)							)
             __SCREAMING_SNAKE_CASE     :List[Any]	       =						tf.tile(tf.expand_dims(_lowerCAmelCase	,1							)	,(1, self.num_choices, 1)							)
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						tf.tile(tf.expand_dims(_lowerCAmelCase	,1							)	,(1, self.num_choices, 1)							)
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						{
                 """input_ids""": multiple_choice_inputs_ids,
                 """attention_mask""": multiple_choice_input_mask,
                 """token_type_ids""": multiple_choice_token_type_ids,
             }
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						model(_lowerCAmelCase							)
             self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.num_choices)							)




       def 					_UpperCamelCase  (		self							)    ->   Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :int	       =						self.prepare_config_and_inputs()
             (
                 __SCREAMING_SNAKE_CASE		
             )     :Dict	       =						config_and_inputs
             __SCREAMING_SNAKE_CASE     :str	       =						{
                 """input_ids""": input_ids,
                 """token_type_ids""": token_type_ids,
                 """langs""": token_type_ids,
                 """lengths""": input_lengths,
             }
             return config, inputs_dict






@require_tf
class       _SCREAMING_SNAKE_CASE(     __lowerCamelCase					,			__lowerCamelCase					,			unittest.TestCase						):
       SCREAMING_SNAKE_CASE_     :   Tuple					=						(
           (
               TFXLMModel,
               TFXLMWithLMHeadModel,
               TFXLMForSequenceClassification,
               TFXLMForQuestionAnsweringSimple,
               TFXLMForTokenClassification,
               TFXLMForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       SCREAMING_SNAKE_CASE_     :   Dict					=						(
           (TFXLMWithLMHeadModel,) if is_tf_available() else ()
       )  # TODO (PVP): Check other models whether language generation is also applicable
       SCREAMING_SNAKE_CASE_     :   str					=						(
           {
               '''feature-extraction''': TFXLMModel,
               '''fill-mask''': TFXLMWithLMHeadModel,
               '''question-answering''': TFXLMForQuestionAnsweringSimple,
               '''text-classification''': TFXLMForSequenceClassification,
               '''text-generation''': TFXLMWithLMHeadModel,
               '''token-classification''': TFXLMForTokenClassification,
               '''zero-shot''': TFXLMForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       SCREAMING_SNAKE_CASE_     :   str					=						False
       SCREAMING_SNAKE_CASE_     :   Union[str, Any]					=						False




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__							)    ->   Optional[int]:





             """simple docstring"""






             if (
                 pipeline_test_casse_name == "QAPipelineTests"
                 and tokenizer_name is not None
                 and not tokenizer_name.endswith('''Fast'''							)
             ):
                   # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                   # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                   # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                   return True

             return False




       def 					_UpperCamelCase  (		self							)    ->   List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :List[Any]	       =						TFXLMModelTester(self							)
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						ConfigTester(self	,config_class=_lowerCAmelCase	,emb_dim=37							)




       def 					_UpperCamelCase  (		self							)    ->   Dict:





             """simple docstring"""






             self.config_tester.run_common_tests()




       def 					_UpperCamelCase  (		self							)    ->   int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Tuple	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_model(*_lowerCAmelCase							)




       def 					_UpperCamelCase  (		self							)    ->   Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :str	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_lm_head(*_lowerCAmelCase							)




       def 					_UpperCamelCase  (		self							)    ->   str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_qa(*_lowerCAmelCase							)




       def 					_UpperCamelCase  (		self							)    ->   Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Tuple	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_sequence_classif(*_lowerCAmelCase							)




       def 					_UpperCamelCase  (		self							)    ->   Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :int	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_for_token_classification(*_lowerCAmelCase							)




       def 					_UpperCamelCase  (		self							)    ->   Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Dict	       =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_xlm_for_multiple_choice(*_lowerCAmelCase							)




       @slow
       def 					_UpperCamelCase  (		self							)    ->   Dict:





             """simple docstring"""






             for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   __SCREAMING_SNAKE_CASE     :int	       =						TFXLMModel.from_pretrained(_lowerCAmelCase							)
                   self.assertIsNotNone(_lowerCAmelCase							)






@require_tf
class       _SCREAMING_SNAKE_CASE(     unittest.TestCase						):




       @slow
       def 					_UpperCamelCase  (		self							)    ->   Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						TFXLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''							)
             __SCREAMING_SNAKE_CASE     :Dict	       =						tf.convert_to_tensor([[14, 4_47]]	,dtype=tf.intaa							)  # the president
             __SCREAMING_SNAKE_CASE     :str	       =						[
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
                 14,
                 4_47,
             ]  # the president the president the president the president the president the president the president the president the president the president
             # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
             __SCREAMING_SNAKE_CASE     :List[str]	       =						model.generate(_lowerCAmelCase	,do_sample=_lowerCAmelCase							)
             self.assertListEqual(output_ids[0].numpy().tolist()	,_lowerCAmelCase							)










"""simple docstring"""
# Lint as: python3

import hashlib
from typing import Union



def   __lowerCamelCase     (							a_  :	Union[str, int, bytes]							)    ->   bytes:
      if isinstance(a_				,     a_							):
            # Data already in bytes, returns as it as
            return hash_data
      elif isinstance(a_				,     a_							):
            # We keep the data as it as for it ot be later encoded to UTF-8
            # However replace `\\` with `/` for Windows compatibility
            __SCREAMING_SNAKE_CASE     :str	       =						hash_data.replace('''\\'''				,     '''/'''							)
      elif isinstance(a_				,     a_							):
            __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						str(a_							)
      else:
            # If data is not of the required type, raise error
            raise InvalidKeyError(a_							)

      return hash_data.encode('''utf-8'''							)






class       _SCREAMING_SNAKE_CASE(     A						):




       def __init__(		self	,SCREAMING_SNAKE_CASE__							)    ->   Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Tuple	       =						'''\nFAILURE TO GENERATE DATASET: Invalid key type detected'''
             __SCREAMING_SNAKE_CASE     :int	       =						f'''\nFound Key {hash_data} of type {type(SCREAMING_SNAKE_CASE__							)}'''
             __SCREAMING_SNAKE_CASE     :Optional[int]	       =						'''\nKeys should be either str, int or bytes type'''
             super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)






class       _SCREAMING_SNAKE_CASE(     A						):




       def __init__(		self	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__	,SCREAMING_SNAKE_CASE__=""							)    ->   List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :str	       =						key
             __SCREAMING_SNAKE_CASE     :str	       =						duplicate_key_indices
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						fix_msg
             __SCREAMING_SNAKE_CASE     :int	       =						'''Found multiple examples generated with the same key'''
             if len(SCREAMING_SNAKE_CASE__							) <= 20:
                   __SCREAMING_SNAKE_CASE     :Dict	       =						f'''\nThe examples at index {", ".join(SCREAMING_SNAKE_CASE__							)} have the key {key}'''
             else:
                   __SCREAMING_SNAKE_CASE     :List[str]	       =						f'''\nThe examples at index {", ".join(duplicate_key_indices[:20]							)}... ({len(SCREAMING_SNAKE_CASE__							) - 20} more) have the key {key}'''
             __SCREAMING_SNAKE_CASE     :Union[str, Any]	       =						'''\n''' + fix_msg if fix_msg else ''''''
             super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)






class       _SCREAMING_SNAKE_CASE:




       def __init__(		self	,SCREAMING_SNAKE_CASE__							)    ->   Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Dict	       =						hashlib.mda(_as_bytes(SCREAMING_SNAKE_CASE__							)							)




       def 					_UpperCamelCase  (		self	,SCREAMING_SNAKE_CASE__							)    ->   int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE     :Tuple	       =						self._split_mda.copy()
             __SCREAMING_SNAKE_CASE     :Optional[Any]	       =						_as_bytes(SCREAMING_SNAKE_CASE__							)
             mda.update(SCREAMING_SNAKE_CASE__							)
             # Convert to integer with hexadecimal conversion
             return int(mda.hexdigest()	,16							)