




from copy import deepcopy
from unittest.case import TestCase

import pytest

from datasets.arrow_dataset import Dataset
from datasets.features import Audio, ClassLabel, Features, Image, Sequence, Value
from datasets.info import DatasetInfo
from datasets.tasks import (
    AudioClassification,
    AutomaticSpeechRecognition,
    ImageClassification,
    LanguageModeling,
    QuestionAnsweringExtractive,
    Summarization,
    TextClassification,
    task_template_from_dict,
)
from datasets.utils.py_utils import asdict


__SCREAMING_SNAKE_CASE       ={
    """id""": """5733be284776f41900661182""",
    """title""": """University_of_Notre_Dame""",
    """context""": """Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.""",
    """question""": """To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?""",
    """answers""": {"""text""": ["""Saint Bernadette Soubirous"""], """answer_start""": [515]},
}

@pytest.mark.parametrize(
    'task_cls'     ,				[
        AudioClassification,
        AutomaticSpeechRecognition,
        ImageClassification,
        LanguageModeling,
        QuestionAnsweringExtractive,
        Summarization,
        TextClassification,
    ]     ,				)
def 			lowerCAmelCase_			(  lowerCamelCase:       List[Any]						)					->			int:


      '''simple docstring'''




      lowerCamelCase__        =				task_cls()
      lowerCamelCase__        =				asdict(lowerCamelCase						)
      lowerCamelCase__        =				task_template_from_dict(lowerCamelCase						)
      assert task == reloaded






class 			A			:
     """simple docstring"""

     def  __snake_case   (    self  :   Tuple			)				->       Optional[Any]:
           lowerCamelCase__        =				LanguageModeling(text_column='input_text'			)
           assert {"input_text": "text"} == task.column_mapping




     def  __snake_case   (    self  :   Tuple			)				->       Optional[Any]:
           lowerCamelCase__        =				Features({'text': Value('string'			)}			)
           lowerCamelCase__        =				{'text_column': 'input_text'}
           lowerCamelCase__        =				LanguageModeling.from_dict(UpperCamelCase__			)
           assert "language-modeling" == task.task
           assert input_schema == task.input_schema






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   List[Any]			)				->       Tuple:
           lowerCamelCase__        =				sorted(['pos', 'neg']			)




     def  __snake_case   (    self  :   str			)				->       int:
           lowerCamelCase__        =				TextClassification(text_column='input_text'	,  label_column='input_label'			)
           self.assertDictEqual({'input_text': 'text', 'input_label': 'labels'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   int			)				->       Optional[int]:
           lowerCamelCase__        =				Features({'text': Value('string'			)}			)
           # Labels are cast to tuple during `TextClassification.__post_init__`, so we do the same here
           lowerCamelCase__        =				Features({'labels': ClassLabel}			)
           lowerCamelCase__        =				{'text_column': 'input_text', 'label_column': 'input_labels'}
           lowerCamelCase__        =				TextClassification.from_dict(UpperCamelCase__			)
           self.assertEqual('text-classification'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)




     def  __snake_case   (    self  :   List[Any]			)				->       Optional[Any]:
           lowerCamelCase__        =				TextClassification(text_column='input_text'	,  label_column='input_label'			)
           self.assertEqual(task.label_schema['labels']	,  UpperCamelCase__			)
           lowerCamelCase__        =				task.align_with_features(Features({'input_label': ClassLabel(names=self.labels			)}			)			)
           self.assertEqual(task.label_schema['labels']	,  ClassLabel(names=self.labels			)			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   List[Any]			)				->       int:
           lowerCamelCase__        =				QuestionAnsweringExtractive(
               context_column='input_context'	,  question_column='input_question'	,  answers_column='input_answers'			)
           self.assertDictEqual(
               {'input_context': 'context', 'input_question': 'question', 'input_answers': 'answers'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   Tuple			)				->       List[Any]:
           lowerCamelCase__        =				Features({'question': Value('string'			), 'context': Value('string'			)}			)
           lowerCamelCase__        =				Features(
               {
                   'answers': Sequence(
                       {
                           'text': Value('string'			),
                           'answer_start': Value('int32'			),
                       }			)
               }			)
           lowerCamelCase__        =				{
               'context_column': 'input_input_context',
               'question_column': 'input_question',
               'answers_column': 'input_answers',
           }
           lowerCamelCase__        =				QuestionAnsweringExtractive.from_dict(UpperCamelCase__			)
           self.assertEqual('question-answering-extractive'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   Optional[int]			)				->       List[str]:
           lowerCamelCase__        =				Summarization(text_column='input_text'	,  summary_column='input_summary'			)
           self.assertDictEqual({'input_text': 'text', 'input_summary': 'summary'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   List[Any]			)				->       Optional[Any]:
           lowerCamelCase__        =				Features({'text': Value('string'			)}			)
           lowerCamelCase__        =				Features({'summary': Value('string'			)}			)
           lowerCamelCase__        =				{'text_column': 'input_text', 'summary_column': 'input_summary'}
           lowerCamelCase__        =				Summarization.from_dict(UpperCamelCase__			)
           self.assertEqual('summarization'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   List[str]			)				->       List[str]:
           lowerCamelCase__        =				AutomaticSpeechRecognition(audio_column='input_audio'	,  transcription_column='input_transcription'			)
           self.assertDictEqual({'input_audio': 'audio', 'input_transcription': 'transcription'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   int			)				->       Dict:
           lowerCamelCase__        =				Features({'audio': Audio()}			)
           lowerCamelCase__        =				Features({'transcription': Value('string'			)}			)
           lowerCamelCase__        =				{
               'audio_column': 'input_audio',
               'transcription_column': 'input_transcription',
           }
           lowerCamelCase__        =				AutomaticSpeechRecognition.from_dict(UpperCamelCase__			)
           self.assertEqual('automatic-speech-recognition'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   int			)				->       Optional[int]:
           lowerCamelCase__        =				sorted(['pos', 'neg']			)




     def  __snake_case   (    self  :   List[str]			)				->       List[str]:
           lowerCamelCase__        =				AudioClassification(audio_column='input_audio'	,  label_column='input_label'			)
           self.assertDictEqual({'input_audio': 'audio', 'input_label': 'labels'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   int			)				->       Any:
           lowerCamelCase__        =				Features({'audio': Audio()}			)
           lowerCamelCase__        =				Features({'labels': ClassLabel}			)
           lowerCamelCase__        =				{
               'audio_column': 'input_image',
               'label_column': 'input_label',
           }
           lowerCamelCase__        =				AudioClassification.from_dict(UpperCamelCase__			)
           self.assertEqual('audio-classification'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)




     def  __snake_case   (    self  :   str			)				->       Optional[Any]:
           lowerCamelCase__        =				AudioClassification(audio_column='input_audio'	,  label_column='input_label'			)
           self.assertEqual(task.label_schema['labels']	,  UpperCamelCase__			)
           lowerCamelCase__        =				task.align_with_features(Features({'input_label': ClassLabel(names=self.labels			)}			)			)
           self.assertEqual(task.label_schema['labels']	,  ClassLabel(names=self.labels			)			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   Tuple			)				->       Union[str, Any]:
           lowerCamelCase__        =				sorted(['pos', 'neg']			)




     def  __snake_case   (    self  :   Tuple			)				->       Tuple:
           lowerCamelCase__        =				ImageClassification(image_column='input_image'	,  label_column='input_label'			)
           self.assertDictEqual({'input_image': 'image', 'input_label': 'labels'}	,  task.column_mapping			)




     def  __snake_case   (    self  :   Tuple			)				->       Optional[Any]:
           lowerCamelCase__        =				Features({'image': Image()}			)
           lowerCamelCase__        =				Features({'labels': ClassLabel}			)
           lowerCamelCase__        =				{
               'image_column': 'input_image',
               'label_column': 'input_label',
           }
           lowerCamelCase__        =				ImageClassification.from_dict(UpperCamelCase__			)
           self.assertEqual('image-classification'	,  task.task			)
           self.assertEqual(UpperCamelCase__	,  task.input_schema			)
           self.assertEqual(UpperCamelCase__	,  task.label_schema			)




     def  __snake_case   (    self  :   Optional[Any]			)				->       List[str]:
           lowerCamelCase__        =				ImageClassification(image_column='input_image'	,  label_column='input_label'			)
           self.assertEqual(task.label_schema['labels']	,  UpperCamelCase__			)
           lowerCamelCase__        =				task.align_with_features(Features({'input_label': ClassLabel(names=self.labels			)}			)			)
           self.assertEqual(task.label_schema['labels']	,  ClassLabel(names=self.labels			)			)






class 			A			(		_SCREAMING_SNAKE_CASE		):
     """simple docstring"""

     def  __snake_case   (    self  :   Tuple			)				->       Union[str, Any]:
           lowerCamelCase__        =				DatasetInfo(task_templates=QuestionAnsweringExtractive()			)
           lowerCamelCase__        =				Dataset.from_dict({k: [v] for k, v in SAMPLE_QUESTION_ANSWERING_EXTRACTIVE.items()}	,  info=UpperCamelCase__			)
           assert isinstance(dataset.info.task_templates	,  UpperCamelCase__			)
           assert len(dataset.info.task_templates			) == 1

           def keep_task(A__  :   Any			):
                 return x

           def dont_keep_task(A__  :   Union[str, Any]			):
                 lowerCamelCase__        =				deepcopy(UpperCamelCase__			)
                 lowerCamelCase__        =				0
                 return out

           lowerCamelCase__        =				dataset.map(UpperCamelCase__			)
           assert mapped_dataset.info.task_templates == dataset.info.task_templates
           # reload from cache
           lowerCamelCase__        =				dataset.map(UpperCamelCase__			)
           assert mapped_dataset.info.task_templates == dataset.info.task_templates

           lowerCamelCase__        =				dataset.map(UpperCamelCase__			)
           assert mapped_dataset.info.task_templates == []
           # reload from cache
           lowerCamelCase__        =				dataset.map(UpperCamelCase__			)
           assert mapped_dataset.info.task_templates == []




     def  __snake_case   (    self  :   List[Any]			)				->       Tuple:
           lowerCamelCase__        =				Features({'text': Value('string'			), 'label': ClassLabel(names=('pos', 'neg')			)}			)
           lowerCamelCase__        =				TextClassification(text_column='text'	,  label_column='label'			)
           lowerCamelCase__        =				DatasetInfo(features=UpperCamelCase__	,  task_templates=UpperCamelCase__			)
           lowerCamelCase__        =				Dataset.from_dict({'text': ['A sentence.'], 'label': ['pos']}	,  info=UpperCamelCase__			)

           def process(A__  :   Dict			):
                 return example

           lowerCamelCase__        =				dataset.remove_columns('label'			)
           lowerCamelCase__        =				modified_dataset.map(UpperCamelCase__			)
           assert mapped_dataset.info.task_templates == []





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)






class 			A			:
     """simple docstring"""

     def __init__(    self  :   Optional[int]	,  A__  :   Optional[Any]			)				->       Dict:
           lowerCamelCase__        =				foo




     def __call__(    self  :   Dict			)				->       Optional[int]:
           return self.foo






class 			A			(		datasets.Dataset		):
     """simple docstring"""

     @fingerprint_transform(inplace=A__			)
     def  __snake_case   (    self  :   Optional[Any]	,  A__  :   Tuple	,  *A__  :   List[str]	,  **A__  :   List[Any]			)				->       Dict:
           return DatasetChild(self.data	,  fingerprint=A__			)




     @fingerprint_transform(inplace=A__			)
     def  __snake_case   (    self  :   Any	,  A__  :   Optional[int]	,  *A__  :   List[str]	,  **A__  :   Union[str, Any]			)				->       List[Any]:
           return DatasetChild(self.data	,  fingerprint=A__			)






class 			A			:
     """simple docstring"""

     def __init__(    self  :   List[Any]	,  A__  :   Tuple			)				->       List[Any]:
           lowerCamelCase__        =				callable




     def __call__(    self  :   Dict	,  *A__  :   Dict	,  **A__  :   List[str]			)				->       List[str]:
           if self.callable is not None:
                 return self.callable(*A__	,  **A__			)




     def __getstate__(    self  :   Optional[Any]			)				->       Union[str, Any]:
           raise pickle.PicklingError()






class 			A			(		_lowerCAmelCase		):
     """simple docstring"""

     @require_transformers
     @pytest.mark.integration
     def  __snake_case   (    self  :   str			)				->       int:
           from transformers import AutoTokenizer

           def encode(A__  :   List[Any]			):
                 return tokenizer(A__			)

           # TODO: add hash consistency tests across sessions
           lowerCamelCase__        =				AutoTokenizer.from_pretrained('bert-base-uncased'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(lambda A__			: tokenizer(A__			)			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				AutoTokenizer.from_pretrained('bert-base-cased'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(lambda A__			: tokenizer(A__			)			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				AutoTokenizer.from_pretrained('bert-base-uncased'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(lambda A__			: tokenizer(A__			)			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     @require_transformers
     @pytest.mark.integration
     def  __snake_case   (    self  :   int			)				->       List[str]:
           from transformers import AutoTokenizer

           lowerCamelCase__        =				AutoTokenizer.from_pretrained('gpt2'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           tokenizer('Hello world !'			)  # call once to change the tokenizer's cache
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)




     @require_regex
     def  __snake_case   (    self  :   Dict			)				->       Dict:
           import regex

           lowerCamelCase__        =				regex.Regex('foo'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				regex.Regex('bar'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				regex.Regex('foo'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)






class 			A			(		_lowerCAmelCase		):
     """simple docstring"""

     def  __snake_case   (    self  :   Tuple			)				->       List[Any]:
           def func():
                 return foo

           lowerCamelCase__        =				[0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				[1]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				[0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Optional[int]			)				->       str:
           def func():
                 pass

           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()

           def func():
                 pass

           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)




     def  __snake_case   (    self  :   List[Any]			)				->       Optional[Any]:
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Union[str, Any]			)				->       str:
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Optional[Any]			)				->       Optional[Any]:
           lowerCamelCase__        =				(
               'co_argcount',
               'co_kwonlyargcount',
               'co_nlocals',
               'co_stacksize',
               'co_flags',
               'co_code',
               'co_consts',
               'co_names',
               'co_varnames',
               'co_filename',
               'co_name',
               'co_firstlineno',
               'co_lnotab',
               'co_freevars',
               'co_cellvars',
           )

           def _create_code_pyaa(*A__  :   Any			):
                 if hasattr(args[-3]	,  'encode'			):
                       lowerCamelCase__        =				list(A__			)
                       lowerCamelCase__        =				args[-3].encode()  # co_lnotab
                       lowerCamelCase__        =				args[-1_0].encode()  # co_code
                 if hasattr(A__	,  'co_posonlyargcount'			):
                       if len(A__			) == 1_6:
                             return CodeType(*A__			)
                       elif len(A__			) == 1_5:
                             return CodeType(args[0]	,  0	,  *args[1:]			)
                       return CodeType(args[0]	,  0	,  0	,  *args[1:]			)
                 elif hasattr(A__	,  'co_kwonlyargcount'			):
                       if len(A__			) == 1_6:
                             return CodeType(args[0]	,  *args[2:]			)
                       elif len(A__			) == 1_5:
                             return CodeType(*A__			)
                       return CodeType(args[0]	,  0	,  *args[1:]			)
                 if len(A__			) == 1_6:
                       return CodeType(args[0]	,  *args[3:]			)
                 elif len(A__			) == 1_5:
                       return CodeType(args[0]	,  *args[2:]			)
                 return CodeType(*A__			)

           def create_ipython_func(A__  :   Any	,  A__  :   Tuple			):
                 def func():
                       return returned_obj

                 lowerCamelCase__        =				func.__code__
                 # Use _create_code from dill in order to make it work for different python versions
                 if datasets.config.PY_VERSION < version.parse('3.8'			):
                       lowerCamelCase__        =				_create_code_pyaa(
                           *[getattr(A__	,  A__			) if k != 'co_filename' else co_filename for k in code_args_pyaa]			)
                 else:
                       lowerCamelCase__        =				code.replace(co_filename=A__			)
                 return FunctionType(A__	,  func.__globals__	,  func.__name__	,  func.__defaults__	,  func.__closure__			)

           lowerCamelCase__  ,		lowerCamelCase__        =				'<ipython-input-2-e0383a102aae>', [0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           lowerCamelCase__  ,		lowerCamelCase__        =				'<ipython-input-2-e0383a102aae>', [1]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           lowerCamelCase__  ,		lowerCamelCase__        =				'<ipython-input-5-713f6613acf3>', [0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)

           lowerCamelCase__  ,		lowerCamelCase__        =				os.path.join(gettempdir()	,  'ipykernel_12345'	,  '321456789.py'			), [0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           lowerCamelCase__  ,		lowerCamelCase__        =				os.path.join(gettempdir()	,  'ipykernel_12345'	,  '321456789.py'			), [1]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           lowerCamelCase__  ,		lowerCamelCase__        =				os.path.join(gettempdir()	,  'ipykernel_12345'	,  '654123987.py'			), [0]
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(create_ipython_func(A__	,  A__			)			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Optional[Any]			)				->       List[Any]:
           lowerCamelCase__  ,		lowerCamelCase__        =				[0], [1]

           def func():
                 return foo, bar

           lowerCamelCase__        =				'__main__'

           def globalvars_mocka_side_effect(A__  :   Optional[Any]	,  *A__  :   str	,  **A__  :   Optional[int]			):
                 return {"foo": foo, "bar": bar}

           def globalvars_mocka_side_effect(A__  :   Any	,  *A__  :   Optional[int]	,  **A__  :   Tuple			):
                 return {"bar": bar, "foo": foo}

           with patch('dill.detect.globalvars'	,  side_effect=A__			) as globalvars_mocka:
                 lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count	,  0			)
           with patch('dill.detect.globalvars'	,  side_effect=A__			) as globalvars_mocka:
                 lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count	,  0			)
           self.assertEqual(A__	,  A__			)






class 			A			(		_lowerCAmelCase		):
     """simple docstring"""

     def  __snake_case   (    self  :   str			)				->       List[str]:
           lowerCamelCase__        =				Hasher.hash('hello'			)
           lowerCamelCase__        =				Hasher.hash('hello'			)
           lowerCamelCase__        =				Hasher.hash('there'			)
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   List[str]			)				->       Optional[Any]:
           lowerCamelCase__        =				Hasher.hash(Foo('hello'			)			)
           lowerCamelCase__        =				Hasher.hash(Foo('hello'			)			)
           lowerCamelCase__        =				Hasher.hash(Foo('there'			)			)
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Tuple			)				->       str:
           lowerCamelCase__        =				Hasher()
           for x in ["hello", Foo('hello'			)]:
                 hasher.update(A__			)
           lowerCamelCase__        =				hasher.hexdigest()
           lowerCamelCase__        =				Hasher()
           for x in ["hello", Foo('hello'			)]:
                 hasher.update(A__			)
           lowerCamelCase__        =				hasher.hexdigest()
           lowerCamelCase__        =				Hasher()
           for x in ["there", Foo('there'			)]:
                 hasher.update(A__			)
           lowerCamelCase__        =				hasher.hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     def  __snake_case   (    self  :   Any			)				->       Tuple:
           with self.assertRaises(pickle.PicklingError			):
                 Hasher.hash(UnpicklableCallable(Foo('hello'			)			)			)




     def  __snake_case   (    self  :   Optional[int]			)				->       Optional[Any]:
           lowerCamelCase__        =				'abc'
           lowerCamelCase__        =				[string, string]  # two strings have the same ids
           lowerCamelCase__        =				[string, string]
           lowerCamelCase__        =				json.loads(F"""[\"{string}\", \"{string}\"]"""			)  # two strings have different ids
           self.assertIs(obja[0]	,  A__			)
           self.assertIs(obja[0]	,  obja[1]			)
           self.assertIs(obja[0]	,  A__			)
           self.assertIs(obja[0]	,  obja[1]			)
           self.assertIsNot(obja[0]	,  A__			)
           self.assertIsNot(obja[0]	,  obja[1]			)
           lowerCamelCase__        =				Hasher.hash(A__			)
           lowerCamelCase__        =				Hasher.hash(A__			)
           lowerCamelCase__        =				Hasher.hash(A__			)
           self.assertEqual(A__	,  A__			)
           self.assertEqual(A__	,  A__			)




     @require_tiktoken
     def  __snake_case   (    self  :   Union[str, Any]			)				->       str:
           import tiktoken

           lowerCamelCase__        =				tiktoken.get_encoding('gpt2'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				tiktoken.get_encoding('r50k_base'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				tiktoken.get_encoding('gpt2'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     @require_torch
     def  __snake_case   (    self  :   Optional[Any]			)				->       List[Any]:
           import torch

           lowerCamelCase__        =				torch.tensor([1.0]			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				torch.tensor([2.0]			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				torch.tensor([1.0]			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)




     @require_spacy
     @require_spacy_model('en_core_web_sm'			)
     @require_spacy_model('fr_core_news_sm'			)
     @pytest.mark.integration
     def  __snake_case   (    self  :   Tuple			)				->       List[str]:
           import spacy

           lowerCamelCase__        =				spacy.load('en_core_web_sm'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				spacy.load('fr_core_news_sm'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           lowerCamelCase__        =				spacy.load('en_core_web_sm'			)
           lowerCamelCase__        =				mda(datasets.utils.py_utils.dumps(A__			)			).hexdigest()
           self.assertEqual(A__	,  A__			)
           self.assertNotEqual(A__	,  A__			)

@pytest.mark.integration
def 			lowerCAmelCase_			(  lowerCamelCase:       Path						)					->			Optional[int]:


      '''simple docstring'''




      lowerCamelCase__        =				tmp_path / 'dir1'
      lowerCamelCase__        =				tmp_path / 'dir2'
      dira.mkdir()
      dira.mkdir()
      lowerCamelCase__        =				'script.py'
      lowerCamelCase__        =				dedent(
          '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '						)
      lowerCamelCase__        =				dira / script_filename
      lowerCamelCase__        =				dira / script_filename
      with script_patha.open('w'						) as f:
            f.write(lowerCamelCase						)
      with script_patha.open('w'						) as f:
            f.write(lowerCamelCase						)
      lowerCamelCase__        =				subprocess.check_output(['python', str(lowerCamelCase						)]						)
      lowerCamelCase__        =				subprocess.check_output(['python', str(lowerCamelCase						)]						)
      assert fingerprinta == fingerprinta

def 			lowerCAmelCase_			(  )					->			int:


      '''simple docstring'''




      lowerCamelCase__        =				{'a': [0, 1, 2]}
      lowerCamelCase__        =				DatasetChild(InMemoryTable.from_pydict(lowerCamelCase						)						)
      lowerCamelCase__        =				dataset.funca()._fingerprint
      assert expected_fingerprint == dataset.funca()._fingerprint
      assert expected_fingerprint != dataset.funca()._fingerprint

      with Pool(2						) as p:
            assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
            assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint

def 			lowerCAmelCase_			(  )					->			Union[str, Any]:


      '''simple docstring'''




      lowerCamelCase__        =				{'a': [0, 1, 2]}






      class 			A			(		datasets.Dataset		):
           """simple docstring"""

           @fingerprint_transform(inplace=A__			)
           def  __snake_case   (    self  :   Any	,  A__  :   Union[str, Any]			)				->       List[Any]:
                 return DummyDatasetChild(self.data	,  fingerprint=A__			)
      lowerCamelCase__        =				DummyDatasetChild(InMemoryTable.from_pydict(lowerCamelCase						)						).func()






      class 			A			(		datasets.Dataset		):
           """simple docstring"""

           @fingerprint_transform(inplace=A__	,  version='1.0.0'			)
           def  __snake_case   (    self  :   str	,  A__  :   List[Any]			)				->       List[str]:
                 return DummyDatasetChild(self.data	,  fingerprint=A__			)
      lowerCamelCase__        =				DummyDatasetChild(InMemoryTable.from_pydict(lowerCamelCase						)						).func()






      class 			A			(		datasets.Dataset		):
           """simple docstring"""

           @fingerprint_transform(inplace=A__	,  version='2.0.0'			)
           def  __snake_case   (    self  :   List[str]	,  A__  :   int			)				->       int:
                 return DummyDatasetChild(self.data	,  fingerprint=A__			)
      lowerCamelCase__        =				DummyDatasetChild(InMemoryTable.from_pydict(lowerCamelCase						)						).func()

      assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3


def 			lowerCAmelCase_			(  )					->			int:


      '''simple docstring'''




      lowerCamelCase__        =				Hasher()
      hasher.update(lambda lowerCamelCase						: x						)