



"""simple docstring"""




from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel


@require_tf
class   a__		:
			_lowerCamelCase												=				MarianConfig
			_lowerCamelCase												=				{}
			_lowerCamelCase												=				'gelu'


			def __init__(							self      :							Optional[int],     lowerCAmelCase      :							Any,     lowerCAmelCase      :							str=13,     lowerCAmelCase      :							Union[str, Any]=7,     lowerCAmelCase      :							Optional[int]=True,     lowerCAmelCase      :							Optional[Any]=False,     lowerCAmelCase      :							int=99,     lowerCAmelCase      :							Dict=32,     lowerCAmelCase      :							str=2,     lowerCAmelCase      :							int=4,     lowerCAmelCase      :							List[str]=37,     lowerCAmelCase      :							List[Any]=0.1,     lowerCAmelCase      :							Union[str, Any]=0.1,     lowerCAmelCase      :							int=20,     lowerCAmelCase      :							Union[str, Any]=2,     lowerCAmelCase      :							Dict=1,     lowerCAmelCase      :							List[Any]=0,     )    ->       str:
									lowercase  :      Any					=   parent
									lowercase  :      List[Any]					=   batch_size
									lowercase  :      int					=   seq_length
									lowercase  :      Any					=   is_training
									lowercase  :      Dict					=   use_labels
									lowercase  :      List[Any]					=   vocab_size
									lowercase  :      int					=   hidden_size
									lowercase  :      List[Any]					=   num_hidden_layers
									lowercase  :      List[str]					=   num_attention_heads
									lowercase  :      int					=   intermediate_size

									lowercase  :      str					=   hidden_dropout_prob
									lowercase  :      str					=   attention_probs_dropout_prob
									lowercase  :      Any					=   max_position_embeddings
									lowercase  :      Union[str, Any]					=   eos_token_id
									lowercase  :      Optional[int]					=   pad_token_id
									lowercase  :      Any					=   bos_token_id


			def        lowercase				(							self      :							Any			)    ->       Optional[int]:
									lowercase  :      Tuple					=   ids_tensor([self.batch_size, self.seq_length - 1],     self.vocab_size			)
									lowercase  :      List[Any]					=   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size			),     1			)
									lowercase  :      List[str]					=   tf.concat([input_ids, eos_tensor],     axis=1			)

									lowercase  :      Tuple					=   ids_tensor([self.batch_size, self.seq_length],     self.vocab_size			)

									lowercase  :      int					=   self.config_cls(
									    vocab_size=self.vocab_size,     d_model=self.hidden_size,     encoder_layers=self.num_hidden_layers,     decoder_layers=self.num_hidden_layers,     encoder_attention_heads=self.num_attention_heads,     decoder_attention_heads=self.num_attention_heads,     encoder_ffn_dim=self.intermediate_size,     decoder_ffn_dim=self.intermediate_size,     dropout=self.hidden_dropout_prob,     attention_dropout=self.attention_probs_dropout_prob,     max_position_embeddings=self.max_position_embeddings,     eos_token_ids=[2],     bos_token_id=self.bos_token_id,     pad_token_id=self.pad_token_id,     decoder_start_token_id=self.pad_token_id,     **self.config_updates,     )
									lowercase  :      List[str]					=   prepare_marian_inputs_dict(lowerCAmelCase,     lowerCAmelCase,     lowerCAmelCase			)
									return config, inputs_dict






			def        lowercase				(							self      :							Dict,     lowerCAmelCase      :							int,     lowerCAmelCase      :							int			)    ->       str:
									lowercase  :      List[str]					=   TFMarianModel(config=lowerCAmelCase			).get_decoder()
									lowercase  :      Optional[Any]					=   inputs_dict['input_ids']

									lowercase  :      Tuple					=   input_ids[:1, :]
									lowercase  :      int					=   inputs_dict['attention_mask'][:1, :]
									lowercase  :      List[str]					=   inputs_dict['head_mask']
									lowercase  :      Union[str, Any]					=   1

									# first forward pass
									lowercase  :      Tuple					=   model(lowerCAmelCase,     attention_mask=lowerCAmelCase,     head_mask=lowerCAmelCase,     use_cache=lowerCAmelCase			)

									lowercase  :      List[Any]					=   outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									lowercase  :      Any					=   ids_tensor((self.batch_size, 3),     config.vocab_size			)
									lowercase  :      str					=   tf.cast(ids_tensor((self.batch_size, 3),     2			),     tf.inta			)

									# append to next input_ids and
									lowercase  :      Union[str, Any]					=   tf.concat([input_ids, next_tokens],     axis=-1			)
									lowercase  :      List[Any]					=   tf.concat([attention_mask, next_attn_mask],     axis=-1			)

									lowercase  :      Optional[Any]					=   model(lowerCAmelCase,     attention_mask=lowerCAmelCase			)[0]
									lowercase  :      List[str]					=   model(lowerCAmelCase,     attention_mask=lowerCAmelCase,     past_key_values=lowerCAmelCase			)[0]

									self.parent.assertEqual(next_tokens.shape[1],     output_from_past.shape[1]			)

									# select random slice
									lowercase  :      int					=   int(ids_tensor((1,),     output_from_past.shape[-1]			)			)
									lowercase  :      List[Any]					=   output_from_no_past[:, -3:, random_slice_idx]
									lowercase  :      Optional[int]					=   output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(lowerCAmelCase,     lowerCAmelCase,     rtol=1e-3			)




def   lowercase__     (			_UpperCAmelCase	,      _UpperCAmelCase	,      _UpperCAmelCase	,      _UpperCAmelCase=None	,      _UpperCAmelCase=None	,      _UpperCAmelCase=None	,      _UpperCAmelCase=None	,      _UpperCAmelCase=None	,      )     ->			Optional[Any]:






						'''simple docstring'''


						if attention_mask is None:
												lowercase  :      Union[str, Any]					=   tf.cast(tf.math.not_equal(_UpperCAmelCase	,      config.pad_token_id				)	,      tf.inta				)
						if decoder_attention_mask is None:
												lowercase  :      Optional[Any]					=   tf.concat(
												    [
												        tf.ones(decoder_input_ids[:, :1].shape	,      dtype=tf.inta				),
												        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]	,      config.pad_token_id				)	,      tf.inta				),
												    ]	,      axis=-1	,      )
						if head_mask is None:
												lowercase  :      List[str]					=   tf.ones((config.encoder_layers, config.encoder_attention_heads)				)
						if decoder_head_mask is None:
												lowercase  :      Tuple					=   tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
						if cross_attn_head_mask is None:
												lowercase  :      Optional[int]					=   tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
						return {
						    "input_ids": input_ids,
						    "decoder_input_ids": decoder_input_ids,
						    "attention_mask": attention_mask,
						    "decoder_attention_mask": decoder_attention_mask,
						    "head_mask": head_mask,
						    "decoder_head_mask": decoder_head_mask,
						    "cross_attn_head_mask": cross_attn_head_mask,
						}


@require_tf
class   a__		(						SCREAMING_SNAKE_CASE__,  SCREAMING_SNAKE_CASE__,  unittest.TestCase     ):
			_lowerCamelCase												=				(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
			_lowerCamelCase												=				(TFMarianMTModel,) if is_tf_available() else ()
			_lowerCamelCase												=				(
			    {
			        'conversational': TFMarianMTModel,
			        'feature-extraction': TFMarianModel,
			        'summarization': TFMarianMTModel,
			        'text2text-generation': TFMarianMTModel,
			        'translation': TFMarianMTModel,
			    }
			    if is_tf_available()
			    else {}
			)
			_lowerCamelCase												=				True
			_lowerCamelCase												=				False
			_lowerCamelCase												=				False


			def        lowercase				(							self      :							int			)    ->       Tuple:
									lowercase  :      List[Any]					=   TFMarianModelTester(self			)
									lowercase  :      str					=   ConfigTester(self,     config_class=lowerCAmelCase			)


			def        lowercase				(							self      :							str			)    ->       Optional[Any]:
									self.config_tester.run_common_tests()






			def        lowercase				(							self      :							List[str]			)    ->       List[str]:
									lowercase  :      List[Any]					=   self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase			)




@require_tf
class   a__		(						unittest.TestCase     ):
			_lowerCamelCase												=				1_000  # show more chars for failing integration tests


			@classmethod
			def        lowercase				(							cls      :							Optional[int]			)    ->       None:
									lowercase  :      int					=   f'''Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'''
									return cls


			@cached_property
			def        lowercase				(							self      :							Optional[Any]			)    ->       MarianTokenizer:
									return AutoTokenizer.from_pretrained(self.model_name			)


			@property
			def        lowercase				(							self      :							Dict			)    ->       int:
									return self.tokenizer.eos_token_id


			@cached_property
			def        lowercase				(							self      :							str			)    ->       str:
									warnings.simplefilter('error'			)
									lowercase  :      TFMarianMTModel					=   TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name			)
									assert isinstance(lowerCAmelCase,     lowerCAmelCase			)
									lowercase  :      Dict					=   model.config
									self.assertListEqual(c.bad_words_ids,     [[c.pad_token_id]]			)
									self.assertEqual(c.max_length,     512			)
									self.assertEqual(c.decoder_start_token_id,     c.pad_token_id			)
									return model


			def        lowercase				(							self      :							Dict,     **lowerCAmelCase      :							Tuple			)    ->       List[str]:
									lowercase  :      Dict					=   self.translate_src_text(**lowerCAmelCase			)
									self.assertListEqual(self.expected_text,     lowerCAmelCase			)






			def        lowercase				(							self      :							Optional[int],     **lowerCAmelCase      :							Optional[Any]			)    ->       List[str]:
									lowercase  :      Tuple					=   self.tokenizer(self.src_text,     **lowerCAmelCase,     padding=lowerCAmelCase,     return_tensors='tf'			)
									lowercase  :      List[Any]					=   self.model.generate(
									    model_inputs.input_ids,     attention_mask=model_inputs.attention_mask,     num_beams=2,     max_length=128			)
									lowercase  :      List[Any]					=   self.tokenizer.batch_decode(generated_ids.numpy(),     skip_special_tokens=lowerCAmelCase			)
									return generated_words




@require_sentencepiece
@require_tokenizers
@require_tf
class   a__		(						SCREAMING_SNAKE_CASE__     ):
			_lowerCamelCase												=				'mt'
			_lowerCamelCase												=				'en'
			_lowerCamelCase												=				['Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.']
			_lowerCamelCase												=				['Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.']


			@unittest.skip('Skipping until #12647 is resolved.'			)
			@slow
			def        lowercase				(							self      :							Tuple			)    ->       int:
									self._assert_generated_batch_equal_expected()




@require_sentencepiece
@require_tokenizers
@require_tf
class   a__		(						SCREAMING_SNAKE_CASE__     ):
			_lowerCamelCase												=				'en'
			_lowerCamelCase												=				'zh'
			_lowerCamelCase												=				['My name is Wolfgang and I live in Berlin']
			_lowerCamelCase												=				['我叫沃尔夫冈 我住在柏林']


			@unittest.skip('Skipping until #12647 is resolved.'			)
			@slow
			def        lowercase				(							self      :							Optional[Any]			)    ->       Optional[int]:
									self._assert_generated_batch_equal_expected()




@require_sentencepiece
@require_tokenizers
@require_tf
class   a__		(						SCREAMING_SNAKE_CASE__     ):
			_lowerCamelCase												=				'en'
			_lowerCamelCase												=				'ROMANCE'
			_lowerCamelCase												=				[
			    '>>fr<< Don\'t spend so much time watching TV.',
			    '>>pt<< Your message has been sent.',
			    '>>es<< He\'s two years older than me.',
			]
			_lowerCamelCase												=				[
			    'Ne passez pas autant de temps à regarder la télé.',
			    'A sua mensagem foi enviada.',
			    'Es dos años más viejo que yo.',
			]


			@unittest.skip('Skipping until #12647 is resolved.'			)
			@slow
			def        lowercase				(							self      :							Tuple			)    ->       List[str]:
									self._assert_generated_batch_equal_expected()






			@unittest.skip('Skipping until #12647 is resolved.'			)
			@slow
			def        lowercase				(							self      :							Dict			)    ->       Union[str, Any]:
									lowercase  :      Optional[int]					=   TranslationPipeline(self.model,     self.tokenizer,     framework='tf'			)
									lowercase  :      Optional[int]					=   pipeline(self.src_text			)
									self.assertEqual(self.expected_text,     [x['translation_text'] for x in output]			)





"""simple docstring"""




import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin


class   a__		(						SCREAMING_SNAKE_CASE__     ):
   _lowerCamelCase												=				'AutoFeatureExtractor'
   _lowerCamelCase												=				'Speech2Text2Tokenizer'


   def __init__(							self      :							str,     lowerCAmelCase      :							List[str],     lowerCAmelCase      :							Optional[Any]			)    ->       Dict:
         super().__init__(lowerCAmelCase,     lowerCAmelCase			)
         lowercase  :      Tuple					=   self.feature_extractor
         lowercase  :      Any					=   False


   def __call__(							self      :							Tuple,     *lowerCAmelCase      :							Tuple,     **lowerCAmelCase      :							Tuple			)    ->       Dict:
         # For backward compatibility
         if self._in_target_context_manager:
               return self.current_processor(*lowerCAmelCase,     **lowerCAmelCase			)

         if "raw_speech" in kwargs:
               warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'			)
               lowercase  :      str					=   kwargs.pop('raw_speech'			)
         else:
               lowercase  :      str					=   kwargs.pop('audio',     lowerCAmelCase			)
         lowercase  :      Dict					=   kwargs.pop('sampling_rate',     lowerCAmelCase			)
         lowercase  :      Optional[Any]					=   kwargs.pop('text',     lowerCAmelCase			)
         if len(lowerCAmelCase			) > 0:
               lowercase  :      Any					=   args[0]
               lowercase  :      int					=   args[1:]

         if audio is None and text is None:
               raise ValueError('You need to specify either an `audio` or `text` input to process.'			)

         if audio is not None:
               lowercase  :      List[Any]					=   self.feature_extractor(lowerCAmelCase,     *lowerCAmelCase,     sampling_rate=lowerCAmelCase,     **lowerCAmelCase			)
         if text is not None:
               lowercase  :      str					=   self.tokenizer(lowerCAmelCase,     **lowerCAmelCase			)

         if text is None:
               return inputs
         elif audio is None:
               return encodings
         else:
               lowercase  :      Tuple					=   encodings['input_ids']
               return inputs


   def        lowercase				(							self      :							str,     *lowerCAmelCase      :							List[str],     **lowerCAmelCase      :							Union[str, Any]			)    ->       str:
         return self.tokenizer.batch_decode(*lowerCAmelCase,     **lowerCAmelCase			)


   def        lowercase				(							self      :							Tuple,     *lowerCAmelCase      :							Union[str, Any],     **lowerCAmelCase      :							int			)    ->       Dict:
         return self.tokenizer.decode(*lowerCAmelCase,     **lowerCAmelCase			)






   @contextmanager
   def        lowercase				(							self      :							str			)    ->       Any:
         warnings.warn(
             '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
             'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
             'your audio inputs, or in a separate call.'			)
         lowercase  :      Optional[Any]					=   True
         lowercase  :      Any					=   self.tokenizer
         yield
         lowercase  :      Any					=   self.feature_extractor
         lowercase  :      Any					=   False
