
from sklearn.metrics import roc_auc_score

import datasets


_lowerCamelCase :       str     					=    '''
This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model\'s predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance.

This metric has three separate use cases:
    - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation.
    - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label.
    - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label.
'''

_lowerCamelCase :       Union[str, Any]     					=    '''
Args:
- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case:
    - binary: expects an array-like of shape (n_samples,)
    - multiclass: expects an array-like of shape (n_samples,)
    - multilabel: expects an array-like of shape (n_samples, n_classes)
- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case:
    - binary: expects an array-like of shape (n_samples,)
    - multiclass: expects an array-like of shape (n_samples, n_classes)
    - multilabel: expects an array-like of shape (n_samples, n_classes)
- average (`str`): Type of average, and is ignored in the binary use case. Defaults to \'macro\'. Options are:
    - `\'micro\'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case.
    - `\'macro\'`: Calculate metrics for each label, and find their unweighted mean.  This does not take label imbalance into account.
    - `\'weighted\'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label).
    - `\'samples\'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case.
    - `None`:  No average is calculated, and scores for each class are returned. Only works with the multilabels use case.
- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None.
- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`.
- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are:
    - `\'ovr\'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == \'macro\'`, because class imbalance affects the composition of each of the \'rest\' groupings.
    - `\'ovo\'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == \'macro\'`.
- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in
    `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in
    `prediction_scores` is used. Defaults to `None`.
Returns:
    roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average=\'None\'`. Otherwise, returns `float`.
Examples:
    Example 1:
        >>> roc_auc_score = datasets.load_metric("roc_auc")
        >>> refs = [1, 0, 1, 1, 0, 0]
        >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores)
        >>> print(round(results[\'roc_auc\'], 2))
        0.78

    Example 2:
        >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass")
        >>> refs = [1, 0, 1, 2, 2, 0]
        >>> pred_scores = [[0.3, 0.5, 0.2],
        ...                 [0.7, 0.2, 0.1],
        ...                 [0.005, 0.99, 0.005],
        ...                 [0.2, 0.3, 0.5],
        ...                 [0.1, 0.1, 0.8],
        ...                 [0.1, 0.7, 0.2]]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class=\'ovr\')
        >>> print(round(results[\'roc_auc\'], 2))
        0.85

    Example 3:
        >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel")
        >>> refs = [[1, 1, 0],
        ...         [1, 1, 0],
        ...         [0, 1, 0],
        ...         [0, 0, 1],
        ...         [0, 1, 1],
        ...         [1, 0, 1]]
        >>> pred_scores = [[0.3, 0.5, 0.2],
        ...                 [0.7, 0.2, 0.1],
        ...                 [0.005, 0.99, 0.005],
        ...                 [0.2, 0.3, 0.5],
        ...                 [0.1, 0.1, 0.8],
        ...                 [0.1, 0.7, 0.2]]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None)
        >>> print([round(res, 2) for res in results[\'roc_auc\']])
        [0.83, 0.38, 0.94]
'''

_lowerCamelCase :       Optional[Any]     					=    '''\
@article{doi:10.1177/0272989X8900900307,
author = {Donna Katzman McClish},
title ={Analyzing a Portion of the ROC Curve},
journal = {Medical Decision Making},
volume = {9},
number = {3},
pages = {190-195},
year = {1989},
doi = {10.1177/0272989X8900900307},
    note ={PMID: 2668680},
URL = {https://doi.org/10.1177/0272989X8900900307},
eprint = {https://doi.org/10.1177/0272989X8900900307}
}


@article{10.1023/A:1010920819831,
author = {Hand, David J. and Till, Robert J.},
title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems},
year = {2001},
issue_date = {November 2001},
publisher = {Kluwer Academic Publishers},
address = {USA},
volume = {45},
number = {2},
issn = {0885-6125},
url = {https://doi.org/10.1023/A:1010920819831},
doi = {10.1023/A:1010920819831},
journal = {Mach. Learn.},
month = {oct},
pages = {171–186},
numpages = {16},
keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic}
}


@article{scikit-learn,
title={Scikit-learn: Machine Learning in {P}ython},
author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
journal={Journal of Machine Learning Research},
volume={12},
pages={2825--2830},
year={2011}
}
'''




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,_KWARGS_DESCRIPTION      )
class      SCREAMING_SNAKE_CASE__  (			datasets.Metric      ):





	'''simple docstring'''



	def 				A							(						self			:					List[Any]			):


				'''simple docstring'''


				return datasets.MetricInfo(
				    description=_DESCRIPTION    ,						citation=_CITATION    ,						inputs_description=_KWARGS_DESCRIPTION    ,						features=datasets.Features(
				        {
				            'prediction_scores': datasets.Sequence(datasets.Value('float'			)			),
				            'references': datasets.Value('int32'			),
				        }
				        if self.config_name == 'multiclass'
				        else {
				            'references': datasets.Sequence(datasets.Value('int32'			)			),
				            'prediction_scores': datasets.Sequence(datasets.Value('float'			)			),
				        }
				        if self.config_name == 'multilabel'
				        else {
				            'references': datasets.Value('int32'			),
				            'prediction_scores': datasets.Value('float'			),
				        }			)    ,						reference_urls=['https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html']    ,						)





	def 				A							(						self			:					Union[str, Any]    ,						lowercase			:					Optional[int]    ,						lowercase			:					List[str]    ,						lowercase			:					Tuple="macro"    ,						lowercase			:					Optional[Any]=None    ,						lowercase			:					List[str]=None    ,						lowercase			:					Any="raise"    ,						lowercase			:					Optional[int]=None    ,						):


				'''simple docstring'''


				return {
				    "roc_auc": roc_auc_score(
				        lowercase    ,						lowercase    ,						average=lowercase    ,						sample_weight=lowercase    ,						max_fpr=lowercase    ,						multi_class=lowercase    ,						labels=lowercase    ,						)
				}

from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel




@require_tf
class      SCREAMING_SNAKE_CASE__  :





	'''simple docstring'''



	_UpperCAmelCase :	Optional[int]					     =      MarianConfig
	_UpperCAmelCase :	str					     =      {}
	_UpperCAmelCase :	Union[str, Any]					     =      "gelu"



	def __init__(						self			:					List[Any]    ,						lowercase			:					List[str]    ,						lowercase			:					str=13    ,						lowercase			:					Union[str, Any]=7    ,						lowercase			:					List[Any]=True    ,						lowercase			:					Tuple=False    ,						lowercase			:					Dict=99    ,						lowercase			:					Optional[Any]=32    ,						lowercase			:					Optional[int]=2    ,						lowercase			:					int=4    ,						lowercase			:					List[str]=37    ,						lowercase			:					Tuple=0.1    ,						lowercase			:					Union[str, Any]=0.1    ,						lowercase			:					Optional[int]=20    ,						lowercase			:					List[Any]=2    ,						lowercase			:					Dict=1    ,						lowercase			:					str=0    ,						):


				'''simple docstring'''


				_snake_case						=    parent
				_snake_case						=    batch_size
				_snake_case						=    seq_length
				_snake_case						=    is_training
				_snake_case						=    use_labels
				_snake_case						=    vocab_size
				_snake_case						=    hidden_size
				_snake_case						=    num_hidden_layers
				_snake_case						=    num_attention_heads
				_snake_case						=    intermediate_size

				_snake_case						=    hidden_dropout_prob
				_snake_case						=    attention_probs_dropout_prob
				_snake_case						=    max_position_embeddings
				_snake_case						=    eos_token_id
				_snake_case						=    pad_token_id
				_snake_case						=    bos_token_id



	def 				A							(						self			:					List[Any]			):


				'''simple docstring'''


				_snake_case						=    ids_tensor([self.batch_size, self.seq_length - 1]    ,						self.vocab_size			)
				_snake_case						=    tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size			)    ,						1			)
				_snake_case						=    tf.concat([input_ids, eos_tensor]    ,						axis=1			)

				_snake_case						=    ids_tensor([self.batch_size, self.seq_length]    ,						self.vocab_size			)

				_snake_case						=    self.config_cls(
				    vocab_size=self.vocab_size    ,						d_model=self.hidden_size    ,						encoder_layers=self.num_hidden_layers    ,						decoder_layers=self.num_hidden_layers    ,						encoder_attention_heads=self.num_attention_heads    ,						decoder_attention_heads=self.num_attention_heads    ,						encoder_ffn_dim=self.intermediate_size    ,						decoder_ffn_dim=self.intermediate_size    ,						dropout=self.hidden_dropout_prob    ,						attention_dropout=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						eos_token_ids=[2]    ,						bos_token_id=self.bos_token_id    ,						pad_token_id=self.pad_token_id    ,						decoder_start_token_id=self.pad_token_id    ,						**self.config_updates    ,						)
				_snake_case						=    prepare_marian_inputs_dict(lowercase    ,						lowercase    ,						lowercase			)
				return config, inputs_dict





	def 				A							(						self			:					List[Any]    ,						lowercase			:					List[str]    ,						lowercase			:					Optional[Any]			):


				'''simple docstring'''


				_snake_case						=    TFMarianModel(config=lowercase			).get_decoder()
				_snake_case						=    inputs_dict['input_ids']

				_snake_case						=    input_ids[:1, :]
				_snake_case						=    inputs_dict['attention_mask'][:1, :]
				_snake_case						=    inputs_dict['head_mask']
				_snake_case						=    1

				# first forward pass
				_snake_case						=    model(lowercase    ,						attention_mask=lowercase    ,						head_mask=lowercase    ,						use_cache=lowercase			)

				_snake_case			,      _snake_case						=    outputs.to_tuple()

				# create hypothetical next token and extent to next_input_ids
				_snake_case						=    ids_tensor((self.batch_size, 3)    ,						config.vocab_size			)
				_snake_case						=    tf.cast(ids_tensor((self.batch_size, 3)    ,						2			)    ,						tf.inta			)

				# append to next input_ids and
				_snake_case						=    tf.concat([input_ids, next_tokens]    ,						axis=-1			)
				_snake_case						=    tf.concat([attention_mask, next_attn_mask]    ,						axis=-1			)

				_snake_case						=    model(lowercase    ,						attention_mask=lowercase			)[0]
				_snake_case						=    model(lowercase    ,						attention_mask=lowercase    ,						past_key_values=lowercase			)[0]

				self.parent.assertEqual(next_tokens.shape[1]    ,						output_from_past.shape[1]			)

				# select random slice
				_snake_case						=    int(ids_tensor((1,)    ,						output_from_past.shape[-1]			)			)
				_snake_case						=    output_from_no_past[:, -3:, random_slice_idx]
				_snake_case						=    output_from_past[:, :, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(lowercase    ,						lowercase    ,						rtol=1E-3			)






def  a_	( __lowercase     :			List[str] ,      __lowercase     :			List[str] ,      __lowercase     :			Any ,      __lowercase     :			int=None ,      __lowercase     :			Tuple=None ,      __lowercase     :			str=None ,      __lowercase     :			Tuple=None ,      __lowercase     :			str=None ,      )				->						List[str]:
			if attention_mask is None:
						_snake_case						=    tf.cast(tf.math.not_equal(__lowercase ,      config.pad_token_id				) ,      tf.inta				)
			if decoder_attention_mask is None:
						_snake_case						=    tf.concat(
						    [
						        tf.ones(decoder_input_ids[:, :1].shape ,      dtype=tf.inta				),
						        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:] ,      config.pad_token_id				) ,      tf.inta				),
						    ] ,      axis=-1 ,      )
			if head_mask is None:
						_snake_case						=    tf.ones((config.encoder_layers, config.encoder_attention_heads)				)
			if decoder_head_mask is None:
						_snake_case						=    tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
			if cross_attn_head_mask is None:
						_snake_case						=    tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
			return {
			    "input_ids": input_ids,
			    "decoder_input_ids": decoder_input_ids,
			    "attention_mask": attention_mask,
			    "decoder_attention_mask": decoder_attention_mask,
			    "head_mask": head_mask,
			    "decoder_head_mask": decoder_head_mask,
			    "cross_attn_head_mask": cross_attn_head_mask,
			}




@require_tf
class      SCREAMING_SNAKE_CASE__  (			UpperCAmelCase							,UpperCAmelCase							,unittest.TestCase      ):





	'''simple docstring'''



	_UpperCAmelCase :	int					     =      (TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
	_UpperCAmelCase :	Tuple					     =      (TFMarianMTModel,) if is_tf_available() else ()
	_UpperCAmelCase :	Optional[Any]					     =      (
	    {
	        "conversational": TFMarianMTModel,
	        "feature-extraction": TFMarianModel,
	        "summarization": TFMarianMTModel,
	        "text2text-generation": TFMarianMTModel,
	        "translation": TFMarianMTModel,
	    }
	    if is_tf_available()
	    else {}
	)
	_UpperCAmelCase :	Union[str, Any]					     =      True
	_UpperCAmelCase :	Union[str, Any]					     =      False
	_UpperCAmelCase :	Tuple					     =      False



	def 				A							(						self			:					int			):


				'''simple docstring'''


				_snake_case						=    TFMarianModelTester(self			)
				_snake_case						=    ConfigTester(self    ,						config_class=lowercase			)



	def 				A							(						self			:					Any			):


				'''simple docstring'''


				self.config_tester.run_common_tests()





	def 				A							(						self			:					Tuple			):


				'''simple docstring'''


				_snake_case						=    self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.check_decoder_model_past_large_inputs(*lowercase			)




@require_tf
class      SCREAMING_SNAKE_CASE__  (			unittest.TestCase      ):





	'''simple docstring'''



	_UpperCAmelCase :	List[Any]					     =      1_0_0_0  # show more chars for failing integration tests



	@classmethod
	def 				A							(						cls			:					Tuple			):


				'''simple docstring'''


				_snake_case						=    f'''Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'''
				return cls



	@cached_property
	def 				A							(						self			:					List[str]			):


				'''simple docstring'''


				return AutoTokenizer.from_pretrained(self.model_name			)



	@property
	def 				A							(						self			:					Tuple			):


				'''simple docstring'''


				return self.tokenizer.eos_token_id



	@cached_property
	def 				A							(						self			:					List[Any]			):


				'''simple docstring'''


				warnings.simplefilter('error'			)
				_snake_case						=    TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name			)
				assert isinstance(lowercase    ,						lowercase			)
				_snake_case						=    model.config
				self.assertListEqual(c.bad_words_ids    ,						[[c.pad_token_id]]			)
				self.assertEqual(c.max_length    ,						512			)
				self.assertEqual(c.decoder_start_token_id    ,						c.pad_token_id			)
				return model



	def 				A							(						self			:					Optional[Any]    ,						**lowercase			:					int			):


				'''simple docstring'''


				_snake_case						=    self.translate_src_text(**lowercase			)
				self.assertListEqual(self.expected_text    ,						lowercase			)





	def 				A							(						self			:					Optional[int]    ,						**lowercase			:					List[Any]			):


				'''simple docstring'''


				_snake_case						=    self.tokenizer(self.src_text    ,						**lowercase    ,						padding=lowercase    ,						return_tensors='tf'			)
				_snake_case						=    self.model.generate(
				    model_inputs.input_ids    ,						attention_mask=model_inputs.attention_mask    ,						num_beams=2    ,						max_length=128			)
				_snake_case						=    self.tokenizer.batch_decode(generated_ids.numpy()    ,						skip_special_tokens=lowercase			)
				return generated_words




@require_sentencepiece
@require_tokenizers
@require_tf
class      SCREAMING_SNAKE_CASE__  (			UpperCAmelCase      ):





	'''simple docstring'''



	_UpperCAmelCase :	Union[str, Any]					     =      "mt"
	_UpperCAmelCase :	List[str]					     =      "en"
	_UpperCAmelCase :	str					     =      ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
	_UpperCAmelCase :	Any					     =      ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]



	@unittest.skip('Skipping until #12647 is resolved.'			)
	@slow
	def 				A							(						self			:					Union[str, Any]			):


				'''simple docstring'''


				self._assert_generated_batch_equal_expected()




@require_sentencepiece
@require_tokenizers
@require_tf
class      SCREAMING_SNAKE_CASE__  (			UpperCAmelCase      ):





	'''simple docstring'''



	_UpperCAmelCase :	Tuple					     =      "en"
	_UpperCAmelCase :	Any					     =      "zh"
	_UpperCAmelCase :	int					     =      ["My name is Wolfgang and I live in Berlin"]
	_UpperCAmelCase :	List[str]					     =      ["我叫沃尔夫冈 我住在柏林"]



	@unittest.skip('Skipping until #12647 is resolved.'			)
	@slow
	def 				A							(						self			:					Optional[Any]			):


				'''simple docstring'''


				self._assert_generated_batch_equal_expected()




@require_sentencepiece
@require_tokenizers
@require_tf
class      SCREAMING_SNAKE_CASE__  (			UpperCAmelCase      ):





	'''simple docstring'''



	_UpperCAmelCase :	Any					     =      "en"
	_UpperCAmelCase :	Any					     =      "ROMANCE"
	_UpperCAmelCase :	int					     =      [
	    ">>fr<< Don't spend so much time watching TV.",
	    ">>pt<< Your message has been sent.",
	    ">>es<< He's two years older than me.",
	]
	_UpperCAmelCase :	Tuple					     =      [
	    "Ne passez pas autant de temps à regarder la télé.",
	    "A sua mensagem foi enviada.",
	    "Es dos años más viejo que yo.",
	]



	@unittest.skip('Skipping until #12647 is resolved.'			)
	@slow
	def 				A							(						self			:					List[str]			):


				'''simple docstring'''


				self._assert_generated_batch_equal_expected()





	@unittest.skip('Skipping until #12647 is resolved.'			)
	@slow
	def 				A							(						self			:					List[Any]			):


				'''simple docstring'''


				_snake_case						=    TranslationPipeline(self.model    ,						self.tokenizer    ,						framework='tf'			)
				_snake_case						=    pipeline(self.src_text			)
				self.assertEqual(self.expected_text    ,						[x['translation_text'] for x in output]			)