





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class 		snake_case_      :







	'''simple docstring'''




	def __init__(	self       :    Any				,			__lowerCamelCase       :    Dict      )					->   Any:

						'''simple docstring'''




						__lowercase					=    foo






	def __call__(	self       :    Union[str, Any]      )					->   List[Any]:

						'''simple docstring'''




						return self.foo





class 		snake_case_      (       datasets.Dataset	):







	'''simple docstring'''




	@fingerprint_transform(inplace=__lowerCamelCase      )
	def       UpperCAmelCase (	self       :    Union[str, Any]				,			__lowerCamelCase       :    str				,			*__lowerCamelCase       :    Tuple				,			**__lowerCamelCase       :    Optional[Any]      )					->   Optional[int]:

						'''simple docstring'''




						return DatasetChild(self.data				,			fingerprint=__lowerCamelCase      )






	@fingerprint_transform(inplace=__lowerCamelCase      )
	def       UpperCAmelCase (	self       :    Union[str, Any]				,			__lowerCamelCase       :    Optional[int]				,			*__lowerCamelCase       :    int				,			**__lowerCamelCase       :    List[str]      )					->   Optional[int]:

						'''simple docstring'''




						return DatasetChild(self.data				,			fingerprint=__lowerCamelCase      )





class 		snake_case_      :







	'''simple docstring'''




	def __init__(	self       :    List[str]				,			__lowerCamelCase       :    str      )					->   Any:

						'''simple docstring'''




						__lowercase					=    callable




	def __call__(	self       :    Tuple				,			*__lowerCamelCase       :    Union[str, Any]				,			**__lowerCamelCase       :    Union[str, Any]      )					->   Tuple:

						'''simple docstring'''




						if self.callable is not None:
											return self.callable(*__lowerCamelCase				,			**__lowerCamelCase      )






	def __getstate__(	self       :    List[Any]      )					->   List[str]:

						'''simple docstring'''




						raise pickle.PicklingError()





class 		snake_case_      (       lowercase_	):







	'''simple docstring'''




	@require_transformers
	@pytest.mark.integration
	def       UpperCAmelCase (	self       :    int      )					->   List[Any]:

						'''simple docstring'''




						from transformers import AutoTokenizer

						def encode(__lowerCamelCase       :    Union[str, Any]      ):
											return tokenizer(__lowerCamelCase      )

						# TODO: add hash consistency tests across sessions
						__lowercase					=    AutoTokenizer.from_pretrained('bert-base-uncased'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase      : tokenizer(__lowerCamelCase      )      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    AutoTokenizer.from_pretrained('bert-base-cased'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase      : tokenizer(__lowerCamelCase      )      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    AutoTokenizer.from_pretrained('bert-base-uncased'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase      : tokenizer(__lowerCamelCase      )      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	@require_transformers
	@pytest.mark.integration
	def       UpperCAmelCase (	self       :    Optional[Any]      )					->   List[str]:

						'''simple docstring'''




						from transformers import AutoTokenizer

						__lowercase					=    AutoTokenizer.from_pretrained('gpt2'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						tokenizer('Hello world !'      )  # call once to change the tokenizer's cache
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )






	@require_regex
	def       UpperCAmelCase (	self       :    str      )					->   Union[str, Any]:

						'''simple docstring'''




						import regex

						__lowercase					=    regex.Regex('foo'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    regex.Regex('bar'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    regex.Regex('foo'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )





class 		snake_case_      (       lowercase_	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    List[str]      )					->   Any:

						'''simple docstring'''




						def func():
											return foo

						__lowercase					=    [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    [1]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   Optional[Any]:

						'''simple docstring'''




						def func():
											pass

						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()

						def func():
											pass

						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str      )					->   List[str]:

						'''simple docstring'''




						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([0]      )      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([1]      )      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([0]      )      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Dict      )					->   int:

						'''simple docstring'''




						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([0]      ).__call__      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([1]      ).__call__      )      ).hexdigest()
						__lowercase					=    mda(datasets.utils.py_utils.dumps(Foo([0]      ).__call__      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   Dict:

						'''simple docstring'''




						__lowercase					=    (
						    'co_argcount',
						    'co_kwonlyargcount',
						    'co_nlocals',
						    'co_stacksize',
						    'co_flags',
						    'co_code',
						    'co_consts',
						    'co_names',
						    'co_varnames',
						    'co_filename',
						    'co_name',
						    'co_firstlineno',
						    'co_lnotab',
						    'co_freevars',
						    'co_cellvars',
						)

						def _create_code_pyaa(*__lowerCamelCase       :    Any      ):
											if hasattr(args[-3]				,			'encode'      ):
																__lowercase					=    list(__lowerCamelCase      )
																__lowercase					=    args[-3].encode()  # co_lnotab
																__lowercase					=    args[-10].encode()  # co_code
											if hasattr(__lowerCamelCase				,			'co_posonlyargcount'      ):
																if len(__lowerCamelCase      ) == 16:
																					return CodeType(*__lowerCamelCase      )
																elif len(__lowerCamelCase      ) == 15:
																					return CodeType(args[0]				,			0				,			*args[1:]      )
																return CodeType(args[0]				,			0				,			0				,			*args[1:]      )
											elif hasattr(__lowerCamelCase				,			'co_kwonlyargcount'      ):
																if len(__lowerCamelCase      ) == 16:
																					return CodeType(args[0]				,			*args[2:]      )
																elif len(__lowerCamelCase      ) == 15:
																					return CodeType(*__lowerCamelCase      )
																return CodeType(args[0]				,			0				,			*args[1:]      )
											if len(__lowerCamelCase      ) == 16:
																return CodeType(args[0]				,			*args[3:]      )
											elif len(__lowerCamelCase      ) == 15:
																return CodeType(args[0]				,			*args[2:]      )
											return CodeType(*__lowerCamelCase      )

						def create_ipython_func(__lowerCamelCase       :    Any				,			__lowerCamelCase       :    List[Any]      ):
											def func():
																return returned_obj

											__lowercase					=    func.__code__
											# Use _create_code from dill in order to make it work for different python versions
											if datasets.config.PY_VERSION < version.parse('3.8'      ):
																__lowercase					=    _create_code_pyaa(
																    *[getattr(__lowerCamelCase				,			__lowerCamelCase      ) if k != 'co_filename' else co_filename for k in code_args_pyaa]      )
											else:
																__lowercase					=    code.replace(co_filename=__lowerCamelCase      )
											return FunctionType(__lowerCamelCase				,			func.__globals__				,			func.__name__				,			func.__defaults__				,			func.__closure__      )

						__lowercase	,			__lowercase					=    '<ipython-input-2-e0383a102aae>', [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						__lowercase	,			__lowercase					=    '<ipython-input-2-e0383a102aae>', [1]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						__lowercase	,			__lowercase					=    '<ipython-input-5-713f6613acf3>', [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )

						__lowercase	,			__lowercase					=    os.path.join(gettempdir()				,			'ipykernel_12345'				,			'321456789.py'      ), [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						__lowercase	,			__lowercase					=    os.path.join(gettempdir()				,			'ipykernel_12345'				,			'321456789.py'      ), [1]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						__lowercase	,			__lowercase					=    os.path.join(gettempdir()				,			'ipykernel_12345'				,			'654123987.py'      ), [0]
						__lowercase					=    mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase				,			__lowerCamelCase      )      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )






	def       UpperCAmelCase (	self       :    Tuple      )					->   Dict:

						'''simple docstring'''




						__lowercase	,			__lowercase					=    [0], [1]

						def func():
											return foo, bar

						__lowercase					=    '__main__'

						def globalvars_mocka_side_effect(__lowerCamelCase       :    List[str]				,			*__lowerCamelCase       :    List[str]				,			**__lowerCamelCase       :    List[str]      ):
											return {"foo": foo, "bar": bar}

						def globalvars_mocka_side_effect(__lowerCamelCase       :    int				,			*__lowerCamelCase       :    List[str]				,			**__lowerCamelCase       :    Any      ):
											return {"bar": bar, "foo": foo}

						with patch('dill.detect.globalvars'				,			side_effect=__lowerCamelCase      ) as globalvars_mocka:
											__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
											self.assertGreater(globalvars_mocka.call_count				,			0      )
						with patch('dill.detect.globalvars'				,			side_effect=__lowerCamelCase      ) as globalvars_mocka:
											__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
											self.assertGreater(globalvars_mocka.call_count				,			0      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )





class 		snake_case_      (       lowercase_	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    Any      )					->   str:

						'''simple docstring'''




						__lowercase					=    Hasher.hash('hello'      )
						__lowercase					=    Hasher.hash('hello'      )
						__lowercase					=    Hasher.hash('there'      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    int      )					->   Dict:

						'''simple docstring'''




						__lowercase					=    Hasher.hash(Foo('hello'      )      )
						__lowercase					=    Hasher.hash(Foo('hello'      )      )
						__lowercase					=    Hasher.hash(Foo('there'      )      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str      )					->   List[str]:

						'''simple docstring'''




						__lowercase					=    Hasher()
						for x in ["hello", Foo('hello'      )]:
											hasher.update(__lowerCamelCase      )
						__lowercase					=    hasher.hexdigest()
						__lowercase					=    Hasher()
						for x in ["hello", Foo('hello'      )]:
											hasher.update(__lowerCamelCase      )
						__lowercase					=    hasher.hexdigest()
						__lowercase					=    Hasher()
						for x in ["there", Foo('there'      )]:
											hasher.update(__lowerCamelCase      )
						__lowercase					=    hasher.hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str      )					->   List[Any]:

						'''simple docstring'''




						with self.assertRaises(pickle.PicklingError      ):
											Hasher.hash(UnpicklableCallable(Foo('hello'      )      )      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   Tuple:

						'''simple docstring'''




						__lowercase					=    'abc'
						__lowercase					=    [string, string]  # two strings have the same ids
						__lowercase					=    [string, string]
						__lowercase					=    json.loads(F"[\"{string}\", \"{string}\"]"      )  # two strings have different ids
						self.assertIs(obja[0]				,			__lowerCamelCase      )
						self.assertIs(obja[0]				,			obja[1]      )
						self.assertIs(obja[0]				,			__lowerCamelCase      )
						self.assertIs(obja[0]				,			obja[1]      )
						self.assertIsNot(obja[0]				,			__lowerCamelCase      )
						self.assertIsNot(obja[0]				,			obja[1]      )
						__lowercase					=    Hasher.hash(__lowerCamelCase      )
						__lowercase					=    Hasher.hash(__lowerCamelCase      )
						__lowercase					=    Hasher.hash(__lowerCamelCase      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )




	@require_tiktoken
	def       UpperCAmelCase (	self       :    int      )					->   Optional[int]:

						'''simple docstring'''




						import tiktoken

						__lowercase					=    tiktoken.get_encoding('gpt2'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    tiktoken.get_encoding('r50k_base'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    tiktoken.get_encoding('gpt2'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )




	@require_torch
	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   Tuple:

						'''simple docstring'''




						import torch

						__lowercase					=    torch.tensor([1.0]      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    torch.tensor([2.0]      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    torch.tensor([1.0]      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )






	@require_spacy
	@require_spacy_model('en_core_web_sm'      )
	@require_spacy_model('fr_core_news_sm'      )
	@pytest.mark.integration
	def       UpperCAmelCase (	self       :    Any      )					->   Union[str, Any]:

						'''simple docstring'''




						import spacy

						__lowercase					=    spacy.load('en_core_web_sm'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    spacy.load('fr_core_news_sm'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						__lowercase					=    spacy.load('en_core_web_sm'      )
						__lowercase					=    mda(datasets.utils.py_utils.dumps(__lowerCamelCase      )      ).hexdigest()
						self.assertEqual(__lowerCamelCase				,			__lowerCamelCase      )
						self.assertNotEqual(__lowerCamelCase				,			__lowerCamelCase      )





@pytest.mark.integration
def    SCREAMING_SNAKE_CASE							(			snake_case       )		->						List[Any]:
					__lowercase					=    tmp_path / 'dir1'
					__lowercase					=    tmp_path / 'dir2'
					dira.mkdir()
					dira.mkdir()
					__lowercase					=    'script.py'
					__lowercase					=    dedent(
					    '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '       )
					__lowercase					=    dira / script_filename
					__lowercase					=    dira / script_filename
					with script_patha.open('w'       ) as f:
										f.write(snake_case       )
					with script_patha.open('w'       ) as f:
										f.write(snake_case       )
					__lowercase					=    subprocess.check_output(['python', str(snake_case       )]       )
					__lowercase					=    subprocess.check_output(['python', str(snake_case       )]       )
					assert fingerprinta == fingerprinta



def    SCREAMING_SNAKE_CASE							(			)		->						Tuple:
					__lowercase					=    {'a': [0, 1, 2]}
					__lowercase					=    DatasetChild(InMemoryTable.from_pydict(snake_case       )       )
					__lowercase					=    dataset.funca()._fingerprint
					assert expected_fingerprint == dataset.funca()._fingerprint
					assert expected_fingerprint != dataset.funca()._fingerprint

					with Pool(2       ) as p:
										assert expected_fingerprint == p.apply_async(dataset.funca       ).get()._fingerprint
										assert expected_fingerprint != p.apply_async(dataset.funca       ).get()._fingerprint



def    SCREAMING_SNAKE_CASE							(			)		->						Dict:
					__lowercase					=    {'a': [0, 1, 2]}

					class 		snake_case_      (       datasets.Dataset	):







						'''simple docstring'''




						@fingerprint_transform(inplace=__lowerCamelCase      )
						def       UpperCAmelCase (	self       :    Union[str, Any]				,			__lowerCamelCase       :    List[str]      )					->   List[Any]:

											'''simple docstring'''




											return DummyDatasetChild(self.data				,			fingerprint=__lowerCamelCase      )





					__lowercase					=    DummyDatasetChild(InMemoryTable.from_pydict(snake_case       )       ).func()

					class 		snake_case_      (       datasets.Dataset	):







						'''simple docstring'''




						@fingerprint_transform(inplace=__lowerCamelCase				,			version='1.0.0'      )
						def       UpperCAmelCase (	self       :    Optional[Any]				,			__lowerCamelCase       :    Any      )					->   List[str]:

											'''simple docstring'''




											return DummyDatasetChild(self.data				,			fingerprint=__lowerCamelCase      )





					__lowercase					=    DummyDatasetChild(InMemoryTable.from_pydict(snake_case       )       ).func()

					class 		snake_case_      (       datasets.Dataset	):







						'''simple docstring'''




						@fingerprint_transform(inplace=__lowerCamelCase				,			version='2.0.0'      )
						def       UpperCAmelCase (	self       :    int				,			__lowerCamelCase       :    Union[str, Any]      )					->   Dict:

											'''simple docstring'''




											return DummyDatasetChild(self.data				,			fingerprint=__lowerCamelCase      )





					__lowercase					=    DummyDatasetChild(InMemoryTable.from_pydict(snake_case       )       ).func()

					assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}       ) == 3



def    SCREAMING_SNAKE_CASE							(			)		->						List[Any]:
					__lowercase					=    Hasher()
					hasher.update(lambda snake_case       : x       )











import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
		from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


SCREAMING_SNAKE_CASE_      :			Union[str, Any]		  =      get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_torch
@require_torchaudio
@require_sentencepiece
class 		snake_case_      (       unittest.TestCase	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    List[Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    tempfile.mkdtemp()

			__lowercase					=    ['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
			__lowercase					=    dict(zip(__lowerCamelCase				,			range(len(__lowerCamelCase      )      )      )      )
			__lowercase					=    Path(self.tmpdirname      )
			save_json(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['vocab_file']      )
			if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
					copyfile(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['spm_file']      )

			__lowercase					=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
			tokenizer.save_pretrained(self.tmpdirname      )

			__lowercase					=    {
			    'feature_size': 24,
			    'num_mel_bins': 24,
			    'padding_value': 0.0,
			    'sampling_rate': 16_000,
			    'return_attention_mask': False,
			    'do_normalize': True,
			}
			save_json(__lowerCamelCase				,			save_dir / FEATURE_EXTRACTOR_NAME      )




	def       UpperCAmelCase (	self       :    Dict				,			**__lowerCamelCase       :    Dict      )					->   Union[str, Any]:

			'''simple docstring'''




			return SpeechaTextTokenizer.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str				,			**__lowerCamelCase       :    List[str]      )					->   Tuple:

			'''simple docstring'''




			return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   str:

			'''simple docstring'''




			shutil.rmtree(self.tmpdirname      )




	def       UpperCAmelCase (	self       :    str      )					->   Any:

			'''simple docstring'''




			__lowercase					=    self.get_tokenizer()
			__lowercase					=    self.get_feature_extractor()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			processor.save_pretrained(self.tmpdirname      )
			__lowercase					=    SpeechaTextProcessor.from_pretrained(self.tmpdirname      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    List[str]      )					->   Optional[Any]:

			'''simple docstring'''




			__lowercase					=    SpeechaTextProcessor(
			    tokenizer=self.get_tokenizer()				,			feature_extractor=self.get_feature_extractor()      )
			processor.save_pretrained(self.tmpdirname      )

			__lowercase					=    self.get_tokenizer(bos_token='(BOS)'				,			eos_token='(EOS)'      )
			__lowercase					=    self.get_feature_extractor(do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			__lowercase					=    SpeechaTextProcessor.from_pretrained(
			    self.tmpdirname				,			bos_token='(BOS)'				,			eos_token='(EOS)'				,			do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer_add_kwargs.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor_add_kwargs.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Dict      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    floats_list((3, 1_000)      )

			__lowercase					=    feature_extractor(__lowerCamelCase				,			return_tensors='np'      )
			__lowercase					=    processor(__lowerCamelCase				,			return_tensors='np'      )

			for key in input_feat_extract.keys():
					self.assertAlmostEqual(input_feat_extract[key].sum()				,			input_processor[key].sum()				,			delta=1E-2      )




	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    'This is a test string'

			__lowercase					=    processor(text=__lowerCamelCase      )

			__lowercase					=    tokenizer(__lowerCamelCase      )

			for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]				,			encoded_processor[key]      )




	def       UpperCAmelCase (	self       :    str      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

			__lowercase					=    processor.batch_decode(__lowerCamelCase      )
			__lowercase					=    tokenizer.batch_decode(__lowerCamelCase      )

			self.assertListEqual(__lowerCamelCase				,			__lowerCamelCase      )






	def       UpperCAmelCase (	self       :    Tuple      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			self.assertListEqual(
			    processor.model_input_names				,			feature_extractor.model_input_names				,			msg='`processor` and `feature_extractor` model input names do not match'				,			)




