
from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel







@require_tf
class 							UpperCAmelCase	:
					a__: str			  =				MarianConfig
					a__: Dict			  =				{}
					a__: Union[str, Any]			  =				"""gelu"""

					def __init__(			self      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Dict=13 ,      lowerCAmelCase      :					Union[str, Any]=7 ,      lowerCAmelCase      :					List[str]=True ,      lowerCAmelCase      :					Tuple=False ,      lowerCAmelCase      :					Union[str, Any]=99 ,      lowerCAmelCase      :					Dict=32 ,      lowerCAmelCase      :					List[Any]=2 ,      lowerCAmelCase      :					Dict=4 ,      lowerCAmelCase      :					int=37 ,      lowerCAmelCase      :					Dict=0.1 ,      lowerCAmelCase      :					Any=0.1 ,      lowerCAmelCase      :					str=20 ,      lowerCAmelCase      :					Dict=2 ,      lowerCAmelCase      :					Optional[int]=1 ,      lowerCAmelCase      :					Tuple=0 ,      ):
											lowercase       :    Optional[Any]											=  parent
											lowercase       :    List[str]											=  batch_size
											lowercase       :    Optional[int]											=  seq_length
											lowercase       :    Union[str, Any]											=  is_training
											lowercase       :    Union[str, Any]											=  use_labels
											lowercase       :    str											=  vocab_size
											lowercase       :    Any											=  hidden_size
											lowercase       :    Tuple											=  num_hidden_layers
											lowercase       :    Any											=  num_attention_heads
											lowercase       :    Optional[int]											=  intermediate_size

											lowercase       :    Any											=  hidden_dropout_prob
											lowercase       :    Optional[Any]											=  attention_probs_dropout_prob
											lowercase       :    Optional[int]											=  max_position_embeddings
											lowercase       :    List[str]											=  eos_token_id
											lowercase       :    str											=  pad_token_id
											lowercase       :    str											=  bos_token_id

					def        _lowerCAmelCase	(			self      :					Optional[Any]					):
											lowercase       :    List[Any]											=  ids_tensor([self.batch_size, self.seq_length - 1] ,      self.vocab_size					)
											lowercase       :    List[str]											=  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size					) ,      1					)
											lowercase       :    str											=  tf.concat([input_ids, eos_tensor] ,      axis=1					)

											lowercase       :    Dict											=  ids_tensor([self.batch_size, self.seq_length] ,      self.vocab_size					)

											lowercase       :    List[Any]											=  self.config_cls(
											    vocab_size=self.vocab_size ,      d_model=self.hidden_size ,      encoder_layers=self.num_hidden_layers ,      decoder_layers=self.num_hidden_layers ,      encoder_attention_heads=self.num_attention_heads ,      decoder_attention_heads=self.num_attention_heads ,      encoder_ffn_dim=self.intermediate_size ,      decoder_ffn_dim=self.intermediate_size ,      dropout=self.hidden_dropout_prob ,      attention_dropout=self.attention_probs_dropout_prob ,      max_position_embeddings=self.max_position_embeddings ,      eos_token_ids=[2] ,      bos_token_id=self.bos_token_id ,      pad_token_id=self.pad_token_id ,      decoder_start_token_id=self.pad_token_id ,      **self.config_updates ,      )
											lowercase       :    int											=  prepare_marian_inputs_dict(lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase					)
											return config, inputs_dict




					def        _lowerCAmelCase	(			self      :					Tuple ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Tuple					):
											lowercase       :    List[Any]											=  TFMarianModel(config=lowerCAmelCase					).get_decoder()
											lowercase       :    Tuple											=  inputs_dict['''input_ids''']

											lowercase       :    int											=  input_ids[:1, :]
											lowercase       :    Union[str, Any]											=  inputs_dict['''attention_mask'''][:1, :]
											lowercase       :    str											=  inputs_dict['''head_mask''']
											lowercase       :    Tuple											=  1

											# first forward pass
											lowercase       :    List[str]											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      head_mask=lowerCAmelCase ,      use_cache=lowerCAmelCase					)

											lowercase				,				lowercase       :    Optional[Any]											=  outputs.to_tuple()

											# create hypothetical next token and extent to next_input_ids
											lowercase       :    str											=  ids_tensor((self.batch_size, 3) ,      config.vocab_size					)
											lowercase       :    int											=  tf.cast(ids_tensor((self.batch_size, 3) ,      2					) ,      tf.inta					)

											# append to next input_ids and
											lowercase       :    Dict											=  tf.concat([input_ids, next_tokens] ,      axis=-1					)
											lowercase       :    Optional[Any]											=  tf.concat([attention_mask, next_attn_mask] ,      axis=-1					)

											lowercase       :    Any											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase					)[0]
											lowercase       :    List[Any]											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      past_key_values=lowerCAmelCase					)[0]

											self.parent.assertEqual(next_tokens.shape[1] ,      output_from_past.shape[1]					)

											# select random slice
											lowercase       :    int											=  int(ids_tensor((1,) ,      output_from_past.shape[-1]					)					)
											lowercase       :    Optional[Any]											=  output_from_no_past[:, -3:, random_slice_idx]
											lowercase       :    int											=  output_from_past[:, :, random_slice_idx]

											# test that outputs are equal for slice
											tf.debugging.assert_near(lowerCAmelCase ,      lowerCAmelCase ,      rtol=1E-3					)





def        lowerCamelCase_      (			UpperCAmelCase_							:				Tuple							,			UpperCAmelCase_							:				Optional[Any]							,			UpperCAmelCase_							:				List[str]							,			UpperCAmelCase_							:				Optional[Any]=None							,			UpperCAmelCase_							:				Union[str, Any]=None							,			UpperCAmelCase_							:				Tuple=None							,			UpperCAmelCase_							:				str=None							,			UpperCAmelCase_							:				int=None							,			):
						if attention_mask is None:
												lowercase       :    Optional[int]											=  tf.cast(tf.math.not_equal(UpperCAmelCase_							,			config.pad_token_id       )							,			tf.inta       )
						if decoder_attention_mask is None:
												lowercase       :    str											=  tf.concat(
												    [
												        tf.ones(decoder_input_ids[:, :1].shape							,			dtype=tf.inta       ),
												        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]							,			config.pad_token_id       )							,			tf.inta       ),
												    ]							,			axis=-1							,			)
						if head_mask is None:
												lowercase       :    Optional[int]											=  tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
						if decoder_head_mask is None:
												lowercase       :    Any											=  tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
						if cross_attn_head_mask is None:
												lowercase       :    Any											=  tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
						return {
						    "input_ids": input_ids,
						    "decoder_input_ids": decoder_input_ids,
						    "attention_mask": attention_mask,
						    "decoder_attention_mask": decoder_attention_mask,
						    "head_mask": head_mask,
						    "decoder_head_mask": decoder_head_mask,
						    "cross_attn_head_mask": cross_attn_head_mask,
						}







@require_tf
class 							UpperCAmelCase	(   __lowerCamelCase						, __lowerCamelCase						, unittest.TestCase					):
					a__: str			  =				(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
					a__: Any			  =				(TFMarianMTModel,) if is_tf_available() else ()
					a__: Optional[Any]			  =				(
					    {
					        """conversational""": TFMarianMTModel,
					        """feature-extraction""": TFMarianModel,
					        """summarization""": TFMarianMTModel,
					        """text2text-generation""": TFMarianMTModel,
					        """translation""": TFMarianMTModel,
					    }
					    if is_tf_available()
					    else {}
					)
					a__: Optional[int]			  =				True
					a__: int			  =				False
					a__: Optional[int]			  =				False

					def        _lowerCAmelCase	(			self      :					Dict					):
											lowercase       :    Dict											=  TFMarianModelTester(self					)
											lowercase       :    Optional[Any]											=  ConfigTester(self ,      config_class=lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					Tuple					):
											self.config_tester.run_common_tests()




					def        _lowerCAmelCase	(			self      :					Dict					):
											lowercase       :    Union[str, Any]											=  self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase					)







@require_tf
class 							UpperCAmelCase	(   unittest.TestCase					):
					a__: Optional[Any]			  =				10_00  # show more chars for failing integration tests

					@classmethod
					def        _lowerCAmelCase	(			cls      :					str					):
											lowercase       :    Optional[int]											=  f'''Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'''
											return cls

					@cached_property
					def        _lowerCAmelCase	(			self      :					str					):
											return AutoTokenizer.from_pretrained(self.model_name					)

					@property
					def        _lowerCAmelCase	(			self      :					int					):
											return self.tokenizer.eos_token_id

					@cached_property
					def        _lowerCAmelCase	(			self      :					int					):
											warnings.simplefilter('''error'''					)
											lowercase       :    TFMarianMTModel											=  TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name					)
											assert isinstance(lowerCAmelCase ,      lowerCAmelCase					)
											lowercase       :    int											=  model.config
											self.assertListEqual(c.bad_words_ids ,      [[c.pad_token_id]]					)
											self.assertEqual(c.max_length ,      512					)
											self.assertEqual(c.decoder_start_token_id ,      c.pad_token_id					)
											return model

					def        _lowerCAmelCase	(			self      :					List[Any] ,      **lowerCAmelCase      :					Dict					):
											lowercase       :    Optional[Any]											=  self.translate_src_text(**lowerCAmelCase					)
											self.assertListEqual(self.expected_text ,      lowerCAmelCase					)




					def        _lowerCAmelCase	(			self      :					List[str] ,      **lowerCAmelCase      :					Union[str, Any]					):
											lowercase       :    List[Any]											=  self.tokenizer(self.src_text ,      **lowerCAmelCase ,      padding=lowerCAmelCase ,      return_tensors='''tf'''					)
											lowercase       :    Any											=  self.model.generate(
											    model_inputs.input_ids ,      attention_mask=model_inputs.attention_mask ,      num_beams=2 ,      max_length=128					)
											lowercase       :    Optional[Any]											=  self.tokenizer.batch_decode(generated_ids.numpy() ,      skip_special_tokens=lowerCAmelCase					)
											return generated_words







@require_sentencepiece
@require_tokenizers
@require_tf
class 							UpperCAmelCase	(   __lowerCamelCase					):
					a__: Any			  =				"""mt"""
					a__: List[Any]			  =				"""en"""
					a__: Union[str, Any]			  =				["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
					a__: str			  =				["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]

					@unittest.skip('''Skipping until #12647 is resolved.'''					)
					@slow
					def        _lowerCAmelCase	(			self      :					List[Any]					):
											self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 							UpperCAmelCase	(   __lowerCamelCase					):
					a__: Tuple			  =				"""en"""
					a__: Tuple			  =				"""zh"""
					a__: Any			  =				["""My name is Wolfgang and I live in Berlin"""]
					a__: List[Any]			  =				["""我叫沃尔夫冈 我住在柏林"""]

					@unittest.skip('''Skipping until #12647 is resolved.'''					)
					@slow
					def        _lowerCAmelCase	(			self      :					Dict					):
											self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 							UpperCAmelCase	(   __lowerCamelCase					):
					a__: Tuple			  =				"""en"""
					a__: Any			  =				"""ROMANCE"""
					a__: Tuple			  =				[
					    """>>fr<< Don't spend so much time watching TV.""",
					    """>>pt<< Your message has been sent.""",
					    """>>es<< He's two years older than me.""",
					]
					a__: Any			  =				[
					    """Ne passez pas autant de temps à regarder la télé.""",
					    """A sua mensagem foi enviada.""",
					    """Es dos años más viejo que yo.""",
					]

					@unittest.skip('''Skipping until #12647 is resolved.'''					)
					@slow
					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											self._assert_generated_batch_equal_expected()




					@unittest.skip('''Skipping until #12647 is resolved.'''					)
					@slow
					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											lowercase       :    Dict											=  TranslationPipeline(self.model ,      self.tokenizer ,      framework='''tf'''					)
											lowercase       :    Optional[Any]											=  pipeline(self.src_text					)
											self.assertEqual(self.expected_text ,      [x['''translation_text'''] for x in output]					)




from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


snake_case__  							=							logging.get_logger(__name__)







class 							UpperCAmelCase	(   __lowerCamelCase					):
					a__: str			  =				["""input_values""", """attention_mask"""]

					def __init__(			self      :					Union[str, Any] ,      lowerCAmelCase      :					Tuple=1 ,      lowerCAmelCase      :					Dict=1_6000 ,      lowerCAmelCase      :					Optional[Any]=128 ,      lowerCAmelCase      :					Dict=1024 ,      lowerCAmelCase      :					List[Any]=0.0 ,      lowerCAmelCase      :					Any=True ,      lowerCAmelCase      :					Tuple=-4.2_67_73_93 ,      lowerCAmelCase      :					Dict=4.5_68_99_74 ,      lowerCAmelCase      :					int=False ,      **lowerCAmelCase      :					Optional[Any] ,      ):
											super().__init__(feature_size=lowerCAmelCase ,      sampling_rate=lowerCAmelCase ,      padding_value=lowerCAmelCase ,      **lowerCAmelCase					)
											lowercase       :    Optional[int]											=  num_mel_bins
											lowercase       :    List[Any]											=  max_length
											lowercase       :    Any											=  do_normalize
											lowercase       :    Any											=  mean
											lowercase       :    Union[str, Any]											=  std
											lowercase       :    int											=  return_attention_mask

					def        _lowerCAmelCase	(			self      :					int ,      lowerCAmelCase      :					np.ndarray ,      lowerCAmelCase      :					int ,      ):
											lowercase       :    List[Any]											=  torch.from_numpy(lowerCAmelCase					).unsqueeze(0					)
											lowercase       :    List[str]											=  ta_kaldi.fbank(
											    lowerCAmelCase ,      htk_compat=lowerCAmelCase ,      sample_frequency=self.sampling_rate ,      use_energy=lowerCAmelCase ,      window_type='''hanning''' ,      num_mel_bins=self.num_mel_bins ,      dither=0.0 ,      frame_shift=10 ,      )

											lowercase       :    Tuple											=  fbank.shape[0]
											lowercase       :    Optional[Any]											=  max_length - n_frames

											# pad or truncate, depending on difference
											if difference > 0:
																	lowercase       :    int											=  torch.nn.ZeroPadad((0, 0, 0, difference)					)
																	lowercase       :    str											=  pad_module(lowerCAmelCase					)
											elif difference < 0:
																	lowercase       :    Optional[int]											=  fbank[0:max_length, :]

											lowercase       :    List[str]											=  fbank.numpy()

											return fbank

					def        _lowerCAmelCase	(			self      :					Tuple ,      lowerCAmelCase      :					np.ndarray					):
											return (input_values - (self.mean)) / (self.std * 2)




					def __call__(			self      :					Any ,      lowerCAmelCase      :					Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]] ,      lowerCAmelCase      :					Optional[int] = None ,      lowerCAmelCase      :					Optional[Union[str, TensorType]] = None ,      **lowerCAmelCase      :					Tuple ,      ):

											if sampling_rate is not None:
																	if sampling_rate != self.sampling_rate:
																							raise ValueError(
																							    f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
																							    f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
																							    f''' {self.sampling_rate} and not {sampling_rate}.'''					)
											else:
																	logger.warning(
																	    '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
																	    '''Failing to do so can result in silent errors that might be hard to debug.'''					)

											lowercase       :    Optional[Any]											=  isinstance(lowerCAmelCase ,      np.ndarray					) and len(raw_speech.shape					) > 1
											if is_batched_numpy and len(raw_speech.shape					) > 2:
																	raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''					)
											lowercase       :    Tuple											=  is_batched_numpy or (
											    isinstance(lowerCAmelCase ,      (list, tuple)					) and (isinstance(raw_speech[0] ,      (np.ndarray, tuple, list)					))
											)

											if is_batched:
																	lowercase       :    Optional[int]											=  [np.asarray(lowerCAmelCase ,      dtype=np.floataa					) for speech in raw_speech]
											elif not is_batched and not isinstance(lowerCAmelCase ,      np.ndarray					):
																	lowercase       :    Dict											=  np.asarray(lowerCAmelCase ,      dtype=np.floataa					)
											elif isinstance(lowerCAmelCase ,      np.ndarray					) and raw_speech.dtype is np.dtype(np.floataa					):
																	lowercase       :    int											=  raw_speech.astype(np.floataa					)

											# always return batch
											if not is_batched:
																	lowercase       :    Dict											=  [raw_speech]

											# extract fbank features and pad/truncate to max_length
											lowercase       :    Dict											=  [self._extract_fbank_features(lowerCAmelCase ,      max_length=self.max_length					) for waveform in raw_speech]

											# convert into BatchFeature
											lowercase       :    Any											=  BatchFeature({'''input_values''': features}					)

											# make sure list is in array format
											lowercase       :    str											=  padded_inputs.get('''input_values'''					)
											if isinstance(input_values[0] ,      lowerCAmelCase					):
																	lowercase       :    Any											=  [np.asarray(lowerCAmelCase ,      dtype=np.floataa					) for feature in input_values]

											# normalization
											if self.do_normalize:
																	lowercase       :    List[str]											=  [self.normalize(lowerCAmelCase					) for feature in input_values]

											if return_tensors is not None:
																	lowercase       :    str											=  padded_inputs.convert_to_tensors(lowerCAmelCase					)

											return padded_inputs


