





import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


__A       : Union[str, Any]						    = {0: "null", 1: "short", 2: "long", 3: "yes", 4: "no"}
__A       : List[Any]						    = set("".join(["‘", "’", "´", "`", ".", ",", "-", "\""]))





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE=0       ,    _SCREAMING_SNAKE_CASE=None							)       ->   Dict:




    """simple docstring"""




    return [" ".join(x.split(' '							)[begin:end]							) for x in answers if len(x.split(' '							)							) > 1]





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE=False							)       ->   List[Any]:




    """simple docstring"""




    if make_sub_answers:
        # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
        # *e.g.* if the correct answer contains a prefix such as "the", or "a"
        UpperCamelCase__    =       (
            given_answers + get_sub_answers(lowercase__       ,    begin=1							) + get_sub_answers(lowercase__       ,    end=-1							)
        )
    UpperCamelCase__    =       []
    for answer in given_answers:
        UpperCamelCase__    =       answer.replace('_'       ,    ' '							).lower()
        UpperCamelCase__    =       "".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ' ' for c in alias							)
        answers.append(' '.join(alias.split()							).strip()							)
    return set(lowercase__							)





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE=1       ,    _SCREAMING_SNAKE_CASE=100							)       ->   int:




    """simple docstring"""




    UpperCamelCase__    =       jax.lax.top_k(lowercase__       ,    lowercase__							)
    UpperCamelCase__    =       jax.lax.top_k(lowercase__       ,    lowercase__							)

    UpperCamelCase__    =       best_end_idx[:, None] - best_start_idx[None, :]
    UpperCamelCase__    =       jnp.logical_or(widths < 0       ,    widths > max_size							)
    UpperCamelCase__    =       (best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
    UpperCamelCase__    =       jnp.argmax(lowercase__							).item()

    return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE							)       ->   Optional[Any]:




    """simple docstring"""




    UpperCamelCase__    =       sample["question"]["text"]
    UpperCamelCase__    =       sample["document"]["tokens"]["token"]
    UpperCamelCase__    =       sample["document"]["tokens"]["is_html"]
    UpperCamelCase__    =       sample["annotations"]["long_answer"]
    UpperCamelCase__    =       sample["annotations"]["short_answers"]

    UpperCamelCase__    =       " ".join([context[i] for i in range(len(lowercase__							)							) if not is_html[i]]							)

    # 0 - No ; 1 - Yes
    for answer in sample["annotations"]["yes_no_answer"]:
        if answer == 0 or answer == 1:
            return {
                "question": question,
                "context": context_string,
                "short": [],
                "long": [],
                "category": "no" if answer == 0 else "yes",
            }

    UpperCamelCase__    =       []
    for s in short_answers:
        short_targets.extend(s['text']							)
    UpperCamelCase__    =       list(set(lowercase__							)							)

    UpperCamelCase__    =       []
    for s in long_answers:
        if s["start_token"] == -1:
            continue
        UpperCamelCase__    =       context[s["start_token"] : s["end_token"]]
        UpperCamelCase__    =       is_html[s["start_token"] : s["end_token"]]
        UpperCamelCase__    =       " ".join([answer[i] for i in range(len(lowercase__							)							) if not html[i]]							)
        if new_answer not in long_targets:
            long_targets.append(lowercase__							)

    UpperCamelCase__    =       "long_short" if len(short_targets + long_targets							) > 0 else "null"

    return {
        "question": question,
        "context": context_string,
        "short": short_targets,
        "long": long_targets,
        "category": category,
    }





def 							__lowerCAmelCase(					)       ->   Dict:




    """simple docstring"""




    UpperCamelCase__    =       load_from_disk('natural-questions-validation'							)
    UpperCamelCase__    =       dataset.map(lowercase__							).remove_columns(['annotations', 'document', 'id']							)
    print(lowercase__							)

    UpperCamelCase__    =       dataset.filter(lambda _SCREAMING_SNAKE_CASE							: (len(x['question']							) + len(x['context']							)) < 4 * 4_096							)
    UpperCamelCase__    =       short_validation_dataset.filter(lambda _SCREAMING_SNAKE_CASE							: x["category"] != "null"							)
    short_validation_dataset

    UpperCamelCase__    =       "vasudevgupta/flax-bigbird-natural-questions"
    UpperCamelCase__    =       FlaxBigBirdForNaturalQuestions.from_pretrained(lowercase__							)
    UpperCamelCase__    =       BigBirdTokenizerFast.from_pretrained(lowercase__							)

    @jax.jit
    def forward(*_SCREAMING_SNAKE_CASE       ,    **_SCREAMING_SNAKE_CASE							):
        UpperCamelCase__    =       model(*lowercase__       ,    **lowercase__							)
        return start_logits, end_logits, jnp.argmax(lowercase__       ,    axis=-1							)

    def evaluate(_SCREAMING_SNAKE_CASE							):
        # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
        UpperCamelCase__    =       tokenizer(
            example['question']       ,    example['context']       ,    return_tensors='np'       ,    max_length=4_096       ,    padding='max_length'       ,    truncation=lowercase__       ,    )

        UpperCamelCase__    =       forward(**lowercase__							)

        UpperCamelCase__    =       CATEGORY_MAPPING[category.item()]

        UpperCamelCase__    =       example["long"] + example["short"]
        if example["category"] in ["yes", "no", "null"]:
            UpperCamelCase__    =       [example["category"]]
        UpperCamelCase__    =       example["category"] != "null"
        # Now target can be: "yes", "no", "null", "list of long & short answers"

        if predicted_category in ["yes", "no", "null"]:
            UpperCamelCase__    =       [predicted_category]
            UpperCamelCase__    =       example["output"] == example["targets"]
            UpperCamelCase__    =       predicted_category != "null"
            return example

        UpperCamelCase__    =       38 if predicted_category == "short" else 1_024
        UpperCamelCase__    =       get_best_valid_start_end_idx(
            start_scores[0]       ,    end_scores[0]       ,    top_k=8       ,    max_size=lowercase__							)

        UpperCamelCase__    =       inputs["input_ids"][0].tolist()
        UpperCamelCase__    =       [tokenizer.decode(input_ids[start_score : end_score + 1]							)]

        UpperCamelCase__    =       expand_to_aliases(example['targets']       ,    make_sub_answers=lowercase__							)
        UpperCamelCase__    =       expand_to_aliases(example['output']							)

        # some preprocessing to both prediction and answer
        UpperCamelCase__    =       {"".join(a.split()							) for a in answers}
        UpperCamelCase__    =       {"".join(p.split()							) for p in predictions}
        UpperCamelCase__    =       {s for s in predictions if s not in ["``", "''", "`", "'"]}

        # if there is a common element, it's a exact match
        UpperCamelCase__    =       len(list(answers & predictions							)							) > 0
        UpperCamelCase__    =       predicted_category != "null" and len(lowercase__							) > 0

        return example

    UpperCamelCase__    =       short_validation_dataset.map(lowercase__							)

    UpperCamelCase__    =       len(lowercase__							)
    UpperCamelCase__    =       len(short_validation_dataset.filter(lambda _SCREAMING_SNAKE_CASE							: x["match"] == 1							)							)
    print('EM score:'       ,    (matched / total) * 100       ,    '%'							)


if __name__ == "__main__":
 main()












from sklearn.metrics import accuracy_score

import datasets


__A       : Union[str, Any]						    = "\nAccuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:\nAccuracy = (TP + TN) / (TP + TN + FP + FN)\n Where:\nTP: True positive\nTN: True negative\nFP: False positive\nFN: False negative\n"


__A       : List[Any]						    = "\nArgs:\n    predictions (`list` of `int`): Predicted labels.\n    references (`list` of `int`): Ground truth labels.\n    normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True.\n    sample_weight (`list` of `float`): Sample weights Defaults to None.\n\nReturns:\n    accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy.\n\nExamples:\n\n    Example 1-A simple example\n        >>> accuracy_metric = datasets.load_metric(\"accuracy\")\n        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])\n        >>> print(results)\n        {'accuracy': 0.5}\n\n    Example 2-The same as Example 1, except with `normalize` set to `False`.\n        >>> accuracy_metric = datasets.load_metric(\"accuracy\")\n        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)\n        >>> print(results)\n        {'accuracy': 3.0}\n\n    Example 3-The same as Example 1, except with `sample_weight` set.\n        >>> accuracy_metric = datasets.load_metric(\"accuracy\")\n        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])\n        >>> print(results)\n        {'accuracy': 0.8778625954198473}\n"


__A       : List[str]						    = "\n@article{scikit-learn,\n  title={Scikit-learn: Machine Learning in {P}ython},\n  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n  journal={Journal of Machine Learning Research},\n  volume={12},\n  pages={2825--2830},\n  year={2011}\n}\n"







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,   _KWARGS_DESCRIPTION  )
class     lowerCamelCase(	datasets.Metric  ):
     '''simple docstring'''


     def 					lowerCAmelCase__		(						self       ):
         return datasets.MetricInfo(
             description=_DESCRIPTION							,    citation=_CITATION							,    inputs_description=_KWARGS_DESCRIPTION							,    features=datasets.Features(
                 {
                     'predictions': datasets.Sequence(datasets.Value('int32'       )       ),
                     'references': datasets.Sequence(datasets.Value('int32'       )       ),
                 }
                 if self.config_name == 'multilabel'
                 else {
                     'predictions': datasets.Value('int32'       ),
                     'references': datasets.Value('int32'       ),
                 }       )							,    reference_urls=['https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html']							,    )




     def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_							,    snake_case_=True							,    snake_case_=None       ):
         return {
             "accuracy": float(
                 accuracy_score(snake_case_							,    snake_case_							,    normalize=snake_case_							,    sample_weight=snake_case_       )       )
         }





