




'''simple docstring'''






from sklearn.metrics import roc_auc_score

import datasets


lowercase	:      List[str]				=       '''
This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model\'s predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance.

This metric has three separate use cases:
    - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation.
    - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label.
    - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label.
'''

lowercase	:      str				=       '''
Args:
- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case:
    - binary: expects an array-like of shape (n_samples,)
    - multiclass: expects an array-like of shape (n_samples,)
    - multilabel: expects an array-like of shape (n_samples, n_classes)
- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case:
    - binary: expects an array-like of shape (n_samples,)
    - multiclass: expects an array-like of shape (n_samples, n_classes)
    - multilabel: expects an array-like of shape (n_samples, n_classes)
- average (`str`): Type of average, and is ignored in the binary use case. Defaults to \'macro\'. Options are:
    - `\'micro\'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case.
    - `\'macro\'`: Calculate metrics for each label, and find their unweighted mean.  This does not take label imbalance into account.
    - `\'weighted\'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label).
    - `\'samples\'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case.
    - `None`:  No average is calculated, and scores for each class are returned. Only works with the multilabels use case.
- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None.
- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`.
- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are:
    - `\'ovr\'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == \'macro\'`, because class imbalance affects the composition of each of the \'rest\' groupings.
    - `\'ovo\'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == \'macro\'`.
- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in
    `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in
    `prediction_scores` is used. Defaults to `None`.
Returns:
    roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average=\'None\'`. Otherwise, returns `float`.
Examples:
    Example 1:
        >>> roc_auc_score = datasets.load_metric("roc_auc")
        >>> refs = [1, 0, 1, 1, 0, 0]
        >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores)
        >>> print(round(results[\'roc_auc\'], 2))
        0.78

    Example 2:
        >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass")
        >>> refs = [1, 0, 1, 2, 2, 0]
        >>> pred_scores = [[0.3, 0.5, 0.2],
        ...                 [0.7, 0.2, 0.1],
        ...                 [0.005, 0.99, 0.005],
        ...                 [0.2, 0.3, 0.5],
        ...                 [0.1, 0.1, 0.8],
        ...                 [0.1, 0.7, 0.2]]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class=\'ovr\')
        >>> print(round(results[\'roc_auc\'], 2))
        0.85

    Example 3:
        >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel")
        >>> refs = [[1, 1, 0],
        ...         [1, 1, 0],
        ...         [0, 1, 0],
        ...         [0, 0, 1],
        ...         [0, 1, 1],
        ...         [1, 0, 1]]
        >>> pred_scores = [[0.3, 0.5, 0.2],
        ...                 [0.7, 0.2, 0.1],
        ...                 [0.005, 0.99, 0.005],
        ...                 [0.2, 0.3, 0.5],
        ...                 [0.1, 0.1, 0.8],
        ...                 [0.1, 0.7, 0.2]]
        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None)
        >>> print([round(res, 2) for res in results[\'roc_auc\']])
        [0.83, 0.38, 0.94]
'''

lowercase	:      int				=       '''\
@article{doi:10.1177/0272989X8900900307,
author = {Donna Katzman McClish},
title ={Analyzing a Portion of the ROC Curve},
journal = {Medical Decision Making},
volume = {9},
number = {3},
pages = {190-195},
year = {1989},
doi = {10.1177/0272989X8900900307},
    note ={PMID: 2668680},
URL = {https://doi.org/10.1177/0272989X8900900307},
eprint = {https://doi.org/10.1177/0272989X8900900307}
}


@article{10.1023/A:1010920819831,
author = {Hand, David J. and Till, Robert J.},
title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems},
year = {2001},
issue_date = {November 2001},
publisher = {Kluwer Academic Publishers},
address = {USA},
volume = {45},
number = {2},
issn = {0885-6125},
url = {https://doi.org/10.1023/A:1010920819831},
doi = {10.1023/A:1010920819831},
journal = {Mach. Learn.},
month = {oct},
pages = {171–186},
numpages = {16},
keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic}
}


@article{scikit-learn,
title={Scikit-learn: Machine Learning in {P}ython},
author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
journal={Journal of Machine Learning Research},
volume={12},
pages={2825--2830},
year={2011}
}
'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,     _KWARGS_DESCRIPTION				)
class   SCREAMING_SNAKE_CASE_		(       datasets.Metric				):



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        return datasets.MetricInfo(
            description=_DESCRIPTION   ,		citation=_CITATION   ,		inputs_description=_KWARGS_DESCRIPTION   ,		features=datasets.Features(
                {
                    '''prediction_scores''': datasets.Sequence(datasets.Value('''float''' ) ),
                    '''references''': datasets.Value('''int32''' ),
                }
                if self.config_name == '''multiclass'''
                else {
                    '''references''': datasets.Sequence(datasets.Value('''int32''' ) ),
                    '''prediction_scores''': datasets.Sequence(datasets.Value('''float''' ) ),
                }
                if self.config_name == '''multilabel'''
                else {
                    '''references''': datasets.Value('''int32''' ),
                    '''prediction_scores''': datasets.Value('''float''' ),
                } )   ,		reference_urls=['''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html''']   ,		)





 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A="macro"   ,		A=None   ,		A=None   ,		A="raise"   ,		A=None   ,		):






        '''simple docstring'''




        return {
            "roc_auc": roc_auc_score(
                A   ,		A   ,		average=A   ,		sample_weight=A   ,		max_fpr=A   ,		multi_class=A   ,		labels=A   ,		)
        }











'''simple docstring'''






import inspect
import unittest

from transformers import DataaVecVisionConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import (
          MODEL_MAPPING,
          DataaVecVisionForImageClassification,
          DataaVecVisionForSemanticSegmentation,
          DataaVecVisionModel,
      )
      from transformers.models.dataavec.modeling_dataavec_vision import DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import BeitImageProcessor

class   SCREAMING_SNAKE_CASE_		:



 def __init__(  self   ,		A   ,		A=100   ,		A=13   ,		A=30   ,		A=2   ,		A=3   ,		A=True   ,		A=True   ,		A=32   ,		A=4   ,		A=4   ,		A=37   ,		A="gelu"   ,		A=0.1   ,		A=0.1   ,		A=10   ,		A=0.0_2   ,		A=3   ,		A=None   ,		A=[0, 1, 2, 3]   ,		):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      parent
        lowercase  :		str						=      100
        lowercase  :		Dict						=      batch_size
        lowercase  :		Tuple						=      image_size
        lowercase  :		int						=      patch_size
        lowercase  :		Optional[int]						=      num_channels
        lowercase  :		Any						=      is_training
        lowercase  :		Dict						=      use_labels
        lowercase  :		Any						=      hidden_size
        lowercase  :		List[Any]						=      num_hidden_layers
        lowercase  :		int						=      num_attention_heads
        lowercase  :		Dict						=      intermediate_size
        lowercase  :		Optional[int]						=      hidden_act
        lowercase  :		Union[str, Any]						=      hidden_dropout_prob
        lowercase  :		str						=      attention_probs_dropout_prob
        lowercase  :		List[str]						=      type_sequence_label_size
        lowercase  :		Dict						=      initializer_range
        lowercase  :		str						=      scope
        lowercase  :		Tuple						=      out_indices
        lowercase  :		Union[str, Any]						=      num_labels

        # in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
        lowercase  :		Union[str, Any]						=      (image_size // patch_size) ** 2
        lowercase  :		List[Any]						=      num_patches + 1



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

        lowercase  :		Union[str, Any]						=      None
        lowercase  :		int						=      None
        if self.use_labels:
               lowercase  :		Dict						=      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size )
               lowercase  :		Tuple						=      ids_tensor([self.batch_size, self.image_size, self.image_size]   ,		self.num_labels )

        lowercase  :		int						=      self.get_config()

        return config, pixel_values, labels, pixel_labels



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        return DataaVecVisionConfig(
            vocab_size=self.vocab_size   ,		image_size=self.image_size   ,		patch_size=self.patch_size   ,		num_channels=self.num_channels   ,		hidden_size=self.hidden_size   ,		num_hidden_layers=self.num_hidden_layers   ,		num_attention_heads=self.num_attention_heads   ,		intermediate_size=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		hidden_dropout_prob=self.hidden_dropout_prob   ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,		is_decoder=A   ,		initializer_range=self.initializer_range   ,		out_indices=self.out_indices   ,		)



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[str]						=      DataaVecVisionModel(config=A )
        model.to(A )
        model.eval()
        lowercase  :		Tuple						=      model(A )
        # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
        lowercase  :		Union[str, Any]						=      (self.image_size // self.patch_size) ** 2
        self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, num_patches + 1, self.hidden_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		Any						=      self.type_sequence_label_size
        lowercase  :		Any						=      DataaVecVisionForImageClassification(A )
        model.to(A )
        model.eval()
        lowercase  :		Dict						=      model(A   ,		labels=A )
        self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.type_sequence_label_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[str]						=      self.num_labels
        lowercase  :		List[Any]						=      DataaVecVisionForSemanticSegmentation(A )
        model.to(A )
        model.eval()
        lowercase  :		int						=      model(A )
        self.parent.assertEqual(
            result.logits.shape   ,		(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )
        lowercase  :		List[str]						=      model(A   ,		labels=A )
        self.parent.assertEqual(
            result.logits.shape   ,		(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )





 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      self.prepare_config_and_inputs()
        lowercase,     lowercase,     lowercase,     lowercase  :		Any						=      config_and_inputs
        lowercase  :		Union[str, Any]						=      {'''pixel_values''': pixel_values}
        return config, inputs_dict






@require_torch
class   SCREAMING_SNAKE_CASE_		(       lowerCamelCase_		,     lowerCamelCase_		,     unittest.TestCase				):
 __snake_case	:     Tuple			   =      (
     (DataaVecVisionModel, DataaVecVisionForImageClassification, DataaVecVisionForSemanticSegmentation)
     if is_torch_available()
     else ()
 )
 __snake_case	:     Optional[Any]			   =      (
     {
         "feature-extraction": DataaVecVisionModel,
         "image-classification": DataaVecVisionForImageClassification,
         "image-segmentation": DataaVecVisionForSemanticSegmentation,
     }
     if is_torch_available()
     else {}
 )

 __snake_case	:     str			   =      False
 __snake_case	:     str			   =      False
 __snake_case	:     int			   =      False



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      DataaVecVisionModelTester(self )
        lowercase  :		List[str]						=      ConfigTester(
            self   ,		config_class=A   ,		has_text_modality=A   ,		hidden_size=37 )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.config_tester.run_common_tests()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        pass



 @require_torch_multi_gpu
 @unittest.skip(
     reason='''Data2VecVision has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`''' )
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        pass



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase  :		int						=      model_class(A )
               self.assertIsInstance(model.get_input_embeddings()   ,		(nn.Module) )
               lowercase  :		Optional[int]						=      model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(A   ,		nn.Linear ) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Dict						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase  :		Tuple						=      model_class(A )
               lowercase  :		List[str]						=      inspect.signature(model.forward )
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               lowercase  :		Tuple						=      [*signature.parameters.keys()]

               lowercase  :		List[Any]						=      ['''pixel_values''']
               self.assertListEqual(arg_names[:1]   ,		A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_segmentation(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        if not self.model_tester.is_training:
               return

        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()
        lowercase  :		Union[str, Any]						=      True

        for model_class in self.all_model_classes:
               if model_class in [*get_values(A )]:
                      continue

               lowercase  :		Optional[int]						=      model_class(A )
               model.to(A )
               model.train()
               lowercase  :		int						=      self._prepare_for_class(A   ,		A   ,		return_labels=A )
               lowercase  :		List[str]						=      model(**A ).loss
               loss.backward()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()
        if not self.model_tester.is_training:
               return

        lowercase  :		List[str]						=      False
        lowercase  :		str						=      True

        for model_class in self.all_model_classes:
               if model_class in [*get_values(A )] or not model_class.supports_gradient_checkpointing:
                      continue
               # TODO: remove the following 3 lines once we have a MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING
               # this can then be incorporated into _prepare_for_class in test_modeling_common.py
               elif model_class.__name__ == "Data2VecVisionForSemanticSegmentation":
                      lowercase,     lowercase,     lowercase,     lowercase  :		Tuple						=      inputs_dict['''pixel_values'''].shape
                      lowercase  :		int						=      torch.zeros(
                          [self.model_tester.batch_size, height, width]   ,		device=A ).long()
               lowercase  :		Optional[Any]						=      model_class(A )
               model.gradient_checkpointing_enable()
               model.to(A )
               model.train()
               lowercase  :		Any						=      self._prepare_for_class(A   ,		A   ,		return_labels=A )
               lowercase  :		Optional[int]						=      model(**A ).loss
               loss.backward()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		List[Any]						=      self.model_tester.prepare_config_and_inputs_for_common()

        lowercase  :		Union[str, Any]						=      _config_zero_init(A )
        for model_class in self.all_model_classes:
               lowercase  :		Dict						=      model_class(config=A )
               for name, param in model.named_parameters():
                      # we skip lambda parameters as these require special initial values
                      # determined by config.layer_scale_init_value
                      if "lambda" in name:
                             continue
                      if param.requires_grad:
                             self.assertIn(
                                 ((param.data.mean() * 1E9).round() / 1E9).item()   ,		[0.0, 1.0]   ,		msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""   ,		)



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A=2E-4   ,		A="outputs"   ,		A=None ):






        '''simple docstring'''




        super().check_pt_tf_outputs(A   ,		A   ,		A   ,		A   ,		A   ,		A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Any						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*A )





 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        for model_name in DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase  :		List[Any]						=      DataaVecVisionModel.from_pretrained(A )
               self.assertIsNotNone(A )






def 		a_( )				->Any:






       '''simple docstring'''




       lowercase  :		Optional[Any]						=      Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''      )
       return image





@require_torch
@require_vision
class   SCREAMING_SNAKE_CASE_		(       unittest.TestCase				):



 @cached_property
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        return (
            BeitImageProcessor.from_pretrained('''facebook/data2vec-vision-base-ft1k''' ) if is_vision_available() else None
        )





 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      DataaVecVisionForImageClassification.from_pretrained('''facebook/data2vec-vision-base-ft1k''' ).to(
            A )

        lowercase  :		List[str]						=      self.default_image_processor
        lowercase  :		List[Any]						=      prepare_img()
        lowercase  :		List[Any]						=      image_processor(images=A   ,		return_tensors='''pt''' ).to(A )

        # forward pass
        with torch.no_grad():
               lowercase  :		List[Any]						=      model(**A )
        lowercase  :		Any						=      outputs.logits

        # verify the logits
        lowercase  :		List[Any]						=      torch.Size((1, 1000) )
        self.assertEqual(logits.shape   ,		A )

        lowercase  :		str						=      torch.tensor([0.3_2_7_7, -0.1_3_9_5, 0.0_9_1_1] ).to(A )

        self.assertTrue(torch.allclose(logits[0, :3]   ,		A   ,		atol=1E-4 ) )

        lowercase  :		Dict						=      [model.config.labelaid[i] for i in ['''remote control, remote''', '''tabby, tabby cat''']]
        self.assertEqual(logits[0].topk(2 ).indices.cpu().tolist()   ,		A )




