
'''simple docstring'''



import functools
from contextlib import contextmanager

import bert_score
from packaging import version

import datasets



@contextmanager
def  lowercase__(		)->   List[str]:






					"""simple docstring"""

					def filter_log(_UpperCamelCase  : Optional[Any]				):
										return False if "This IS expected if you are initializing" in record.msg else True

					_UpperCamelCase											=   datasets.utils.logging.get_logger("transformers.modeling_utils"				)
					logger.addFilter(lowercase__				)
					try:
										yield
					finally:
										logger.removeFilter(lowercase__				)


snake_case_   :  List[str]								=      '''\\n@inproceedings{bert-score,\n  title={BERTScore: Evaluating Text Generation with BERT},\n  author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi},\n  booktitle={International Conference on Learning Representations},\n  year={2020},\n  url={https://openreview.net/forum?id=SkeHuCVFDr}\n}\n'''

snake_case_   :  int								=      '''\\nBERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference\nsentences by cosine similarity.\nIt has been shown to correlate with human judgment on sentence-level and system-level evaluation.\nMoreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language\ngeneration tasks.\n\nSee the project\'s README at https://github.com/Tiiiger/bert_score#readme for more information.\n'''

snake_case_   :  List[Any]								=      '''\nBERTScore Metrics with the hashcode from a source against one or more references.\n\nArgs:\n    predictions (list of str): Prediction/candidate sentences.\n    references (list of str or list of list of str): Reference sentences.\n    lang (str): Language of the sentences; required (e.g. \'en\').\n    model_type (str): Bert specification, default using the suggested\n        model for the target language; has to specify at least one of\n        `model_type` or `lang`.\n    num_layers (int): The layer of representation to use,\n        default using the number of layers tuned on WMT16 correlation data.\n    verbose (bool): Turn on intermediate status update.\n    idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict.\n    device (str): On which the contextual embedding model will be allocated on.\n        If this argument is None, the model lives on cuda:0 if cuda is available.\n    nthreads (int): Number of threads.\n    batch_size (int): Bert score processing batch size,\n        at least one of `model_type` or `lang`. `lang` needs to be\n        specified when `rescale_with_baseline` is True.\n    rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline.\n    baseline_path (str): Customized baseline file.\n    use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10.\n\nReturns:\n    precision: Precision.\n    recall: Recall.\n    f1: F1 score.\n    hashcode: Hashcode of the library.\n\nExamples:\n\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> bertscore = datasets.load_metric(\"bertscore\")\n    >>> results = bertscore.compute(predictions=predictions, references=references, lang=\"en\")\n    >>> print([round(v, 2) for v in results[\"f1\"]])\n    [1.0, 1.0]\n'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,			_KWARGS_DESCRIPTION  )
class     A_ (  datasets.Metric  ):

			'''simple docstring'''





			def       a	(					self   ):
								return datasets.MetricInfo(
								    description=_DESCRIPTION			,     citation=_CITATION			,     homepage="https://github.com/Tiiiger/bert_score"			,     inputs_description=_KWARGS_DESCRIPTION			,     features=datasets.Features(
								        {
								            "predictions": datasets.Value("string"			,     id="sequence"   ),
								            "references": datasets.Sequence(datasets.Value("string"			,     id="sequence"   )			,     id="references"   ),
								        }   )			,     codebase_urls=["https://github.com/Tiiiger/bert_score"]			,     reference_urls=[
								        "https://github.com/Tiiiger/bert_score",
								        "https://arxiv.org/abs/1904.09675",
								    ]			,     )




			def       a	(					self			,     A_			,     A_			,     A_=None			,     A_=None			,     A_=None			,     A_=False			,     A_=False			,     A_=None			,     A_=64			,     A_=4			,     A_=False			,     A_=False			,     A_=None			,     A_=False			,     ):
								_UpperCamelCase											=   bert_score.utils.get_hash
								_UpperCamelCase											=   bert_score.BERTScorer

								if version.parse(bert_score.__version__   ) >= version.parse("0.3.10"   ):
													_UpperCamelCase											=   functools.partial(A_			,     use_fast_tokenizer=A_   )
													_UpperCamelCase											=   functools.partial(A_			,     use_fast_tokenizer=A_   )
								elif use_fast_tokenizer:
													raise ImportWarning(
													    "To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of `bert-score` doesn't match this condition.\n"
													    "You can install it with `pip install \"bert-score>=0.3.10\"`."   )

								if model_type is None:
													assert lang is not None, "either lang or model_type should be specified"
													_UpperCamelCase											=   bert_score.utils.langamodel[lang.lower()]

								if num_layers is None:
													_UpperCamelCase											=   bert_score.utils.modelalayers[model_type]

								_UpperCamelCase											=   get_hash(
								    model=A_			,     num_layers=A_			,     idf=A_			,     rescale_with_baseline=A_			,     use_custom_baseline=baseline_path is not None			,     )

								with filter_logging_context():
													if not hasattr(self			,     "cached_bertscorer"   ) or self.cached_bertscorer.hash != hashcode:
																		_UpperCamelCase											=   scorer(
																		    model_type=A_			,     num_layers=A_			,     batch_size=A_			,     nthreads=A_			,     all_layers=A_			,     idf=A_			,     device=A_			,     lang=A_			,     rescale_with_baseline=A_			,     baseline_path=A_			,     )

								(_UpperCamelCase)											=   self.cached_bertscorer.score(
								    cands=A_			,     refs=A_			,     verbose=A_			,     batch_size=A_			,     )
								_UpperCamelCase											=   {
								    "precision": P.tolist(),
								    "recall": R.tolist(),
								    "f1": F.tolist(),
								    "hashcode": hashcode,
								}
								return output_dict




			def       a	(					self			,     A_=None			,     A_=None			,     **A_   ):







								if references is not None:
													_UpperCamelCase											=   [[ref] if isinstance(A_			,     A_   ) else ref for ref in references]
								super().add_batch(predictions=A_			,     references=A_			,     **A_   )




			def       a	(					self			,     A_=None			,     A_=None			,     **A_   ):







								if isinstance(A_			,     A_   ):
													_UpperCamelCase											=   [reference]
								super().add(prediction=A_			,     reference=A_			,     **A_   )










'''simple docstring'''



import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer

class     A_ (  tf.keras.layers.Layer  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_ = None			,     A_ = None			,     A_ = None			,     A_ = "longest"			,     A_ = True			,     A_ = 5_12			,     A_ = None			,     A_ = True			,     A_ = True			,     A_ = True			,     **A_			,     ):
								super().__init__()
								if use_fast_bert_tokenizer:
													_UpperCamelCase											=   FastBertTokenizer(
													    A_			,     token_out_type=tf.intaa			,     lower_case_nfd_strip_accents=A_			,     **A_   )
								else:
													_UpperCamelCase											=   tf.lookup.StaticVocabularyTable(
													    tf.lookup.KeyValueTensorInitializer(
													        keys=A_			,     key_dtype=tf.string			,     values=tf.range(tf.size(A_			,     out_type=tf.intaa   )			,     dtype=tf.intaa   )			,     value_dtype=tf.intaa			,     )			,     num_oov_buckets=1			,     )
													_UpperCamelCase											=   BertTokenizerLayer(
													    A_			,     token_out_type=tf.intaa			,     lower_case=A_			,     **A_   )

								_UpperCamelCase											=   vocab_list
								_UpperCamelCase											=   do_lower_case
								_UpperCamelCase											=   cls_token_id or vocab_list.index("[CLS]"   )
								_UpperCamelCase											=   sep_token_id or vocab_list.index("[SEP]"   )
								_UpperCamelCase											=   pad_token_id or vocab_list.index("[PAD]"   )
								_UpperCamelCase											=   ShrinkLongestTrimmer(max_length - 3			,     axis=1   )  # Allow room for special tokens
								_UpperCamelCase											=   max_length
								_UpperCamelCase											=   padding
								_UpperCamelCase											=   truncation
								_UpperCamelCase											=   pad_to_multiple_of
								_UpperCamelCase											=   return_token_type_ids
								_UpperCamelCase											=   return_attention_mask




			@classmethod
			def       a	(					cls			,     A_			,     **A_   ):  # noqa: F821
								_UpperCamelCase											=   kwargs.pop("do_lower_case"			,     A_   )
								_UpperCamelCase											=   tokenizer.do_lower_case if do_lower_case is None else do_lower_case
								_UpperCamelCase											=   kwargs.pop("cls_token_id"			,     A_   )
								_UpperCamelCase											=   tokenizer.cls_token_id if cls_token_id is None else cls_token_id
								_UpperCamelCase											=   kwargs.pop("sep_token_id"			,     A_   )
								_UpperCamelCase											=   tokenizer.sep_token_id if sep_token_id is None else sep_token_id
								_UpperCamelCase											=   kwargs.pop("pad_token_id"			,     A_   )
								_UpperCamelCase											=   tokenizer.pad_token_id if pad_token_id is None else pad_token_id

								_UpperCamelCase											=   tokenizer.get_vocab()
								_UpperCamelCase											=   sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,     key=lambda A_   : x[1]   )
								_UpperCamelCase											=   [entry[0] for entry in vocab]
								return cls(
								    vocab_list=A_			,     do_lower_case=A_			,     cls_token_id=A_			,     sep_token_id=A_			,     pad_token_id=A_			,     **A_			,     )




			@classmethod
			def       a	(					cls			,     A_			,     *A_			,     **A_   ):
								try:
													_UpperCamelCase											=   BertTokenizer.from_pretrained(A_			,     *A_			,     **A_   )
								except:  # noqa: E722
													from .tokenization_bert_fast import BertTokenizerFast

													_UpperCamelCase											=   BertTokenizerFast.from_pretrained(A_			,     *A_			,     **A_   )
								return cls.from_tokenizer(A_			,     **A_   )




			def       a	(					self			,     A_   ):
								if self.do_lower_case:
													_UpperCamelCase											=   case_fold_utfa(A_   )
								_UpperCamelCase											=   self.tf_tokenizer.tokenize(A_   )
								return tokens.merge_dims(1			,     -1   )




			def       a	(					self			,     A_			,     A_=None			,     A_=None			,     A_=None			,     A_=None			,     A_=None			,     A_=None			,     A_=None			,     ):
								if padding is None:
													_UpperCamelCase											=   self.padding
								if padding not in ("longest", "max_length"):
													raise ValueError("Padding must be either 'longest' or 'max_length'!"   )
								if max_length is not None and text_pair is not None:
													# Because we have to instantiate a Trimmer to do it properly
													raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"   )
								if max_length is None:
													_UpperCamelCase											=   self.max_length
								if truncation is None:
													_UpperCamelCase											=   self.truncation
								if pad_to_multiple_of is None:
													_UpperCamelCase											=   self.pad_to_multiple_of
								if return_token_type_ids is None:
													_UpperCamelCase											=   self.return_token_type_ids
								if return_attention_mask is None:
													_UpperCamelCase											=   self.return_attention_mask
								if not isinstance(A_			,     tf.Tensor   ):
													_UpperCamelCase											=   tf.convert_to_tensor(A_   )
								if text_pair is not None and not isinstance(A_			,     tf.Tensor   ):
													_UpperCamelCase											=   tf.convert_to_tensor(A_   )
								if text_pair is not None:
													if text.shape.rank > 1:
																		raise ValueError("text argument should not be multidimensional when a text pair is supplied!"   )
													if text_pair.shape.rank > 1:
																		raise ValueError("text_pair should not be multidimensional!"   )
								if text.shape.rank == 2:
													_UpperCamelCase				,	_UpperCamelCase											=   text[:, 0], text[:, 1]
								_UpperCamelCase											=   self.unpaired_tokenize(A_   )
								if text_pair is None:  # Unpaired text
													if truncation:
																		_UpperCamelCase											=   text[:, : max_length - 2]  # Allow room for special tokens
													_UpperCamelCase				,	_UpperCamelCase											=   combine_segments(
													    (text,)			,     start_of_sequence_id=self.cls_token_id			,     end_of_segment_id=self.sep_token_id   )
								else:  # Paired text
													_UpperCamelCase											=   self.unpaired_tokenize(A_   )
													if truncation:
																		_UpperCamelCase				,	_UpperCamelCase											=   self.paired_trimmer.trim([text, text_pair]   )
													_UpperCamelCase				,	_UpperCamelCase											=   combine_segments(
													    (text, text_pair)			,     start_of_sequence_id=self.cls_token_id			,     end_of_segment_id=self.sep_token_id   )
								if padding == "longest":
													_UpperCamelCase											=   input_ids.bounding_shape(axis=1   )
													if pad_to_multiple_of is not None:
																		# No ceiling division in tensorflow, so we negate floordiv instead
																		_UpperCamelCase											=   pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,     A_   ))
								else:
													_UpperCamelCase											=   max_length

								_UpperCamelCase				,	_UpperCamelCase											=   pad_model_inputs(A_			,     max_seq_length=A_			,     pad_value=self.pad_token_id   )
								_UpperCamelCase											=   {"input_ids": input_ids}
								if return_attention_mask:
													_UpperCamelCase											=   attention_mask
								if return_token_type_ids:
													_UpperCamelCase				,	_UpperCamelCase											=   pad_model_inputs(
													    A_			,     max_seq_length=A_			,     pad_value=self.pad_token_id   )
													_UpperCamelCase											=   token_type_ids
								return output




			def       a	(					self   ):
								return {
								    "vocab_list": self.vocab_list,
								    "do_lower_case": self.do_lower_case,
								    "cls_token_id": self.cls_token_id,
								    "sep_token_id": self.sep_token_id,
								    "pad_token_id": self.pad_token_id,
								}



