



import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_lowercase:				List[Any]       =	logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_lowercase:				Any       =	direct_transformers_import(Path(__file__).parent)


_lowercase:				int       =	{
    "AutoTokenizer": "PreTrainedTokenizerBase",
    "AutoFeatureExtractor": "FeatureExtractionMixin",
    "AutoImageProcessor": "ImageProcessingMixin",
}



class 			_lowercase							(     lowerCAmelCase			):





	"""simple docstring"""
	__A					=							["feature_extractor", "tokenizer"]
	# Names need to be attr_class for attr in attributes
	__A					=							None
	__A					=							None
	__A					=							None




	def __init__(self	, *lowerCamelCase_	, **lowerCamelCase_	):
						"""simple docstring"""

						for key in kwargs:
											if key not in self.attributes:
																raise TypeError(F'''Unexpected keyword argument {key}.'''	)
						for arg, attribute_name in zip(lowerCamelCase_	, self.attributes	):
											if attribute_name in kwargs:
																raise TypeError(F'''Got multiple values for argument {attribute_name}.'''	)
											else:
																a        =     arg

						if len(lowerCamelCase_	) != len(self.attributes	):
											raise ValueError(
											    F'''This processor requires {len(self.attributes	)} arguments: {', '.join(self.attributes	)}. Got '''
											    F'''{len(lowerCamelCase_	)} arguments instead.'''	)

						# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
						for attribute_name, arg in kwargs.items():
											a        =     getattr(self	, F'''{attribute_name}_class'''	)
											# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
											a        =     AUTO_TO_BASE_CLASS_MAPPING.get(lowerCamelCase_	, lowerCamelCase_	)
											if isinstance(lowerCamelCase_	, lowerCamelCase_	):
																a        =     tuple(getattr(lowerCamelCase_	, lowerCamelCase_	) for n in class_name if n is not None	)
											else:
																a        =     getattr(lowerCamelCase_	, lowerCamelCase_	)

											if not isinstance(lowerCamelCase_	, lowerCamelCase_	):
																raise ValueError(
																    F'''Received a {type(lowerCamelCase_	).__name__} for argument {attribute_name}, but a {class_name} was expected.'''	)

											setattr(self	, lowerCamelCase_	, lowerCamelCase_	)




	def __repr__(self	):
						"""simple docstring"""

						a        =     [F'''- {name}: {repr(getattr(self	, lowerCamelCase_	)	)}''' for name in self.attributes]
						a        =     "\n".join(lowerCamelCase_	)
						return F'''{self.__class__.__name__}:\n{attributes_repr}'''




	def   UpperCamelCase_ (self	, lowerCamelCase_	, lowerCamelCase_ = False	, **lowerCamelCase_	):
						"""simple docstring"""

						os.makedirs(lowerCamelCase_	, exist_ok=lowerCamelCase_	)

						if push_to_hub:
											a        =     kwargs.pop("commit_message"	, lowerCamelCase_	)
											a        =     kwargs.pop("repo_id"	, save_directory.split(os.path.sep	)[-1]	)
											a        =     self._create_repo(lowerCamelCase_	, **lowerCamelCase_	)
											a        =     self._get_files_timestamps(lowerCamelCase_	)
						# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
						# loaded from the Hub.
						if self._auto_class is not None:
											a        =     [getattr(self	, lowerCamelCase_	) for attribute_name in self.attributes]
											a        =     [(a.init_kwargs if isinstance(lowerCamelCase_	, lowerCamelCase_	) else a) for a in attrs]
											custom_object_save(self	, lowerCamelCase_	, config=lowerCamelCase_	)

						for attribute_name in self.attributes:
											a        =     getattr(self	, lowerCamelCase_	)
											# Include the processor class in the attribute config so this processor can then be reloaded with the
											# `AutoProcessor` API.
											if hasattr(lowerCamelCase_	, "_set_processor_class"	):
																attribute._set_processor_class(self.__class__.__name__	)
											attribute.save_pretrained(lowerCamelCase_	)

						if self._auto_class is not None:
											# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
											for attribute_name in self.attributes:
																a        =     getattr(self	, lowerCamelCase_	)
																if isinstance(lowerCamelCase_	, lowerCamelCase_	):
																					del attribute.init_kwargs["auto_map"]

						if push_to_hub:
											self._upload_modified_files(
											    lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_	, commit_message=lowerCamelCase_	, token=kwargs.get("use_auth_token"	)	, )




	@classmethod
	def   UpperCamelCase_ (cls	, lowerCamelCase_	, lowerCamelCase_ = None	, lowerCamelCase_ = False	, lowerCamelCase_ = False	, lowerCamelCase_ = None	, lowerCamelCase_ = "main"	, **lowerCamelCase_	, ):
						"""simple docstring"""

						a        =     cache_dir
						a        =     force_download
						a        =     local_files_only
						a        =     revision

						a        =     kwargs.pop("use_auth_token"	, lowerCamelCase_	)
						if use_auth_token is not None:
											warnings.warn(
											    "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."	, lowerCamelCase_	)
											if token is not None:
																raise ValueError(
																    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."	)
											a        =     use_auth_token

						if token is not None:
											# change to `token` in a follow-up PR
											a        =     token

						a        =     cls._get_arguments_from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)
						return cls(*lowerCamelCase_	)




	@classmethod
	def   UpperCamelCase_ (cls	, lowerCamelCase_="AutoProcessor"	):
						"""simple docstring"""

						if not isinstance(lowerCamelCase_	, lowerCamelCase_	):
											a        =     auto_class.__name__

						import transformers.models.auto as auto_module

						if not hasattr(lowerCamelCase_	, lowerCamelCase_	):
											raise ValueError(F'''{auto_class} is not a valid auto class.'''	)

						a        =     auto_class




	@classmethod
	def   UpperCamelCase_ (cls	, lowerCamelCase_	, **lowerCamelCase_	):
						"""simple docstring"""

						a        =     []
						for attribute_name in cls.attributes:
											a        =     getattr(cls	, F'''{attribute_name}_class'''	)
											if isinstance(lowerCamelCase_	, lowerCamelCase_	):
																a        =     tuple(getattr(lowerCamelCase_	, lowerCamelCase_	) if n is not None else None for n in class_name	)
																a        =     kwargs.get("use_fast"	, lowerCamelCase_	)
																if use_fast and classes[1] is not None:
																					a        =     classes[1]
																else:
																					a        =     classes[0]
											else:
																a        =     getattr(lowerCamelCase_	, lowerCamelCase_	)

											args.append(attribute_class.from_pretrained(lowerCamelCase_	, **lowerCamelCase_	)	)
						return args




	@property
	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						a        =     getattr(self	, self.attributes[0]	)
						return getattr(lowerCamelCase_	, "model_input_names"	, lowerCamelCase_	)




_lowercase:				Optional[Any]       =	copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
				_lowercase:				Union[str, Any]       =	ProcessorMixin.push_to_hub.__doc__.format(
				    object="processor", object_class="AutoProcessor", object_files="processor files"
				)





from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse("3.8"):
				import importlib_metadata
else:
				import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
_lowercase:				Optional[Any]       =	"\\n"

_lowercase:				List[Any]       =	"\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n"

_lowercase:				Dict       =	"\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'.\n        Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase',\n        'tokenize', 'use_effective_order'.\n    wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'.\n        Keywords include 'concatenate_texts'.\nReturns: depending on the XTREME-S task, one or several of:\n    \"accuracy\": Accuracy - for 'fleurs-lang_id', 'minds14'\n    \"f1\": F1 score - for 'minds14'\n    \"wer\": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'\n    \"cer\": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'\n    \"bleu\": BLEU score according to the `sacrebleu` metric - for 'covost2'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls')  # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel'\n    >>> references = [\"it is sunny here\", \"paper and pen are essentials\"]\n    >>> predictions = [\"it's sunny\", \"paper pen are essential\"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {'wer': 0.56, 'cer': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'covost2')\n    >>> references = [\"bonjour paris\", \"il est necessaire de faire du sport de temps en temp\"]\n    >>> predictions = [\"bonjour paris\", \"il est important de faire du sport souvent\"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {'bleu': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'fleurs-lang_id')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {'accuracy': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'minds14')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {'f1': 0.58, 'accuracy': 0.6}\n"

_lowercase:				List[Any]       =	["fleurs-asr", "mls", "voxpopuli", "babel", "covost2", "fleurs-lang_id", "minds14"]
_lowercase:				Tuple       =	""

try:
				from jiwer import transforms as tr

				_lowercase:				Any       =	True
except ImportError:
				_lowercase:				Optional[int]       =	False

if _jiwer_available and version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"):



				class 			_lowercase							(     tr.AbstractTransform			):





					"""simple docstring"""




					def __init__(self	, lowerCamelCase_ = " "	):
										"""simple docstring"""

										a        =     sentence_delimiter




					def   UpperCamelCase_ (self	, lowerCamelCase_	):
										"""simple docstring"""

										return list(lowerCamelCase_	)




					def   UpperCamelCase_ (self	, lowerCamelCase_	):
										"""simple docstring"""

										a        =     []
										for sent_idx, sentence in enumerate(lowerCamelCase_	):
															chars.extend(self.process_string(lowerCamelCase_	)	)
															if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(lowerCamelCase_	) - 1:
																				chars.append(self.sentence_delimiter	)
										return chars




				_lowercase:				List[Any]       =	tr.Compose(
				    [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
				)
elif _jiwer_available:
				_lowercase:				List[Any]       =	tr.Compose(
				    [
				        tr.RemoveMultipleSpaces(),
				        tr.Strip(),
				        tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
				        tr.ReduceToListOfListOfChars(),
				    ]
				)
else:
				_lowercase:				List[str]       =	None






def 					a(   A :					List[Any]				,  A :					int )	-> Union[str, Any]:
					"""simple docstring"""





					return float((preds == labels).mean() )






def 					a(   A :					int				,  A :					List[str] )	-> str:
					"""simple docstring"""





					return {
					    "f1": float(fa_score(y_true=A				,  y_pred=A				,  average="macro" ) ),
					    "accuracy": simple_accuracy(A				,  A ),
					}






def 					a(   A :					Optional[Any]				,  A :					Dict				,  A :					int="exp"				,  A :					int=None				,  A :					List[Any]=False				,  A :					Tuple=False				,  A :					Tuple=None				,  A :					Tuple=False				,  )	-> Tuple:
					"""simple docstring"""





					a        =     [[label] for label in labels]
					a        =     list(A )
					try:
										import sacrebleu as scb
					except ImportError:
										raise ValueError(
										    "sacrebleu has to be installed in order to apply the bleu metric for covost2."
										    "You can install it via `pip install sacrebleu`." )

					if version.parse(scb.__version__ ) < version.parse("1.4.12" ):
										raise ImportWarning(
										    "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
										    "You can install it with `pip install \"sacrebleu>=1.4.12\"`." )

					a        =     len(labels[0] )
					if any(len(A ) != references_per_prediction for refs in labels ):
										raise ValueError("Sacrebleu requires the same number of references for each prediction" )
					a        =     [[refs[i] for refs in labels] for i in range(A )]
					a        =     scb.corpus_bleu(
					    A				,  A				,  smooth_method=A				,  smooth_value=A				,  force=A				,  lowercase=A				,  use_effective_order=A				,  **({"tokenize": tokenize} if tokenize else {})				,  )
					return {"bleu": output.score}






def 					a(   A :					str				,  A :					Tuple				,  A :					Optional[Any]				,  A :					str )	-> int:
					"""simple docstring"""





					try:
										from jiwer import compute_measures
					except ImportError:
										raise ValueError(
										    f'''jiwer has to be installed in order to apply the wer metric for {config_name}.'''
										    "You can install it via `pip install jiwer`." )

					if concatenate_texts:
										a        =     compute_measures(A				,  A )["wer"]

										a        =     compute_measures(A				,  A				,  truth_transform=A				,  hypothesis_transform=A )["wer"]
										return {"wer": wer, "cer": cer}
					else:

										def compute_score(A :					List[Any]				,  A :					Any				,  A :					Union[str, Any]="wer" ):
															a        =     0
															a        =     0
															for prediction, reference in zip(A				,  A ):
																				if score_type == "wer":
																									a        =     compute_measures(A				,  A )
																				elif score_type == "cer":
																									a        =     compute_measures(
																									    A				,  A				,  truth_transform=A				,  hypothesis_transform=A )
																				incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
																				total += measures["substitutions"] + measures["deletions"] + measures["hits"]
															return incorrect / total

										return {"wer": compute_score(A				,  A				,  "wer" ), "cer": compute_score(A				,  A				,  "cer" )}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,   _KWARGS_DESCRIPTION			)
class 			_lowercase							(     datasets.Metric			):





	"""simple docstring"""




	def   UpperCamelCase_ (self	):
						"""simple docstring"""

						if self.config_name not in _CONFIG_NAMES:
											raise KeyError(F'''You should supply a configuration name selected in {_CONFIG_NAMES}'''	)

						a        =     "int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string"

						return datasets.MetricInfo(
						    description=_DESCRIPTION	, citation=_CITATION	, inputs_description=_KWARGS_DESCRIPTION	, features=datasets.Features(
						        {"predictions": datasets.Value(lowerCamelCase_	), "references": datasets.Value(lowerCamelCase_	)}	)	, codebase_urls=[]	, reference_urls=[]	, format="numpy"	, )




	def   UpperCamelCase_ (self	, lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_=None	, lowerCamelCase_=None	):
						"""simple docstring"""

						a        =     bleu_kwargs if bleu_kwargs is not None else {}
						a        =     wer_kwargs if wer_kwargs is not None else {}

						if self.config_name == "fleurs-lang_id":
											return {"accuracy": simple_accuracy(lowerCamelCase_	, lowerCamelCase_	)}
						elif self.config_name == "minds14":
											return fa_and_simple_accuracy(lowerCamelCase_	, lowerCamelCase_	)
						elif self.config_name == "covost2":
											a        =     bleu_kwargs.pop("smooth_method"	, "exp"	)
											a        =     bleu_kwargs.pop("smooth_value"	, lowerCamelCase_	)
											a        =     bleu_kwargs.pop("force"	, lowerCamelCase_	)
											a        =     bleu_kwargs.pop("lowercase"	, lowerCamelCase_	)
											a        =     bleu_kwargs.pop("tokenize"	, lowerCamelCase_	)
											a        =     bleu_kwargs.pop("use_effective_order"	, lowerCamelCase_	)
											return bleu(
											    preds=lowerCamelCase_	, labels=lowerCamelCase_	, smooth_method=lowerCamelCase_	, smooth_value=lowerCamelCase_	, force=lowerCamelCase_	, lowercase=lowerCamelCase_	, tokenize=lowerCamelCase_	, use_effective_order=lowerCamelCase_	, )
						elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
											a        =     wer_kwargs.pop("concatenate_texts"	, lowerCamelCase_	)
											return wer_and_cer(lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_	, self.config_name	)
						else:
											raise KeyError(F'''You should supply a configuration name selected in {_CONFIG_NAMES}'''	)
