






"""simple docstring"""




import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)






class  UpperCamelCase    :


	def __init__(  self	:		int							,		UpperCAmelCase__	:		Union[str, Any]							)      -> int:
			_a						: Union[str, Any]    =			foo







	def __call__(  self	:		str							)      -> List[str]:
			return self.foo







class  UpperCamelCase    (			datasets.Dataset   ):


	@fingerprint_transform(inplace=UpperCAmelCase__							)
	def _lowercase			(  self	:		List[str]							,		UpperCAmelCase__	:		int							,		*UpperCAmelCase__	:		Optional[Any]							,		**UpperCAmelCase__	:		Optional[int]							)      -> Dict:
			return DatasetChild(self.data							,		fingerprint=UpperCAmelCase__							)







	@fingerprint_transform(inplace=UpperCAmelCase__							)
	def _lowercase			(  self	:		List[str]							,		UpperCAmelCase__	:		Dict							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Dict							)      -> str:
			return DatasetChild(self.data							,		fingerprint=UpperCAmelCase__							)







class  UpperCamelCase    :


	def __init__(  self	:		Dict							,		UpperCAmelCase__	:		Dict							)      -> List[Any]:
			_a						: Union[str, Any]    =			callable


	def __call__(  self	:		List[str]							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		List[str]							)      -> Any:
			if self.callable is not None:
					return self.callable(*UpperCAmelCase__							,		**UpperCAmelCase__							)







	def __getstate__(  self	:		Optional[int]							)      -> Dict:
			raise pickle.PicklingError()







class  UpperCamelCase    (			snake_case_   ):


	@require_transformers
	@pytest.mark.integration
	def _lowercase			(  self	:		Dict							)      -> Optional[Any]:
			from transformers import AutoTokenizer

			def encode(UpperCAmelCase__	:		List[str]							):
					return tokenizer(UpperCAmelCase__							)

			# TODO: add hash consistency tests across sessions
			_a						: int    =			AutoTokenizer.from_pretrained("""bert-base-uncased"""							)
			_a						: int    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__							: tokenizer(UpperCAmelCase__							)							)							).hexdigest()
			_a						: Optional[int]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: List[Any]    =			AutoTokenizer.from_pretrained("""bert-base-cased"""							)
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Dict    =			mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__							: tokenizer(UpperCAmelCase__							)							)							).hexdigest()
			_a						: Union[str, Any]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Any    =			AutoTokenizer.from_pretrained("""bert-base-uncased"""							)
			_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__							: tokenizer(UpperCAmelCase__							)							)							).hexdigest()
			_a						: List[str]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	@require_transformers
	@pytest.mark.integration
	def _lowercase			(  self	:		Dict							)      -> Optional[int]:
			from transformers import AutoTokenizer

			_a						: int    =			AutoTokenizer.from_pretrained("""gpt2"""							)
			_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			tokenizer("""Hello world !"""							)  # call once to change the tokenizer's cache
			_a						: int    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







	@require_regex
	def _lowercase			(  self	:		Optional[Any]							)      -> List[Any]:
			import regex

			_a						: Optional[int]    =			regex.Regex("""foo"""							)
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Tuple    =			regex.Regex("""bar"""							)
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Tuple    =			regex.Regex("""foo"""							)
			_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







class  UpperCamelCase    (			snake_case_   ):


	def _lowercase			(  self	:		Dict							)      -> List[str]:
			def func():
					return foo

			_a						: Tuple    =			[0]
			_a						: Optional[int]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: List[str]    =			[1]
			_a						: str    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: str    =			[0]
			_a						: List[str]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		Union[str, Any]							)      -> List[Any]:
			def func():
					pass

			_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()

			def func():
					pass

			_a						: Dict    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		Dict							)      -> int:
			_a						: int    =			mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
			_a						: List[str]    =			mda(datasets.utils.py_utils.dumps(Foo([1]							)							)							).hexdigest()
			_a						: str    =			mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		str							)      -> Optional[Any]:
			_a						: Optional[Any]    =			mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
			_a						: int    =			mda(datasets.utils.py_utils.dumps(Foo([1]							).__call__							)							).hexdigest()
			_a						: Union[str, Any]    =			mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		Tuple							)      -> int:
			_a						: Optional[int]    =			(
			    """co_argcount""",
			    """co_kwonlyargcount""",
			    """co_nlocals""",
			    """co_stacksize""",
			    """co_flags""",
			    """co_code""",
			    """co_consts""",
			    """co_names""",
			    """co_varnames""",
			    """co_filename""",
			    """co_name""",
			    """co_firstlineno""",
			    """co_lnotab""",
			    """co_freevars""",
			    """co_cellvars""",
			)

			def _create_code_pyaa(*UpperCAmelCase__	:		Tuple							):
					if hasattr(args[-3]							,		"""encode"""							):
							_a						: Tuple    =			list(UpperCAmelCase__							)
							_a						: List[Any]    =			args[-3].encode()  # co_lnotab
							_a						: List[str]    =			args[-10].encode()  # co_code
					if hasattr(UpperCAmelCase__							,		"""co_posonlyargcount"""							):
							if len(UpperCAmelCase__							) == 16:
									return CodeType(*UpperCAmelCase__							)
							elif len(UpperCAmelCase__							) == 15:
									return CodeType(args[0]							,		0							,		*args[1:]							)
							return CodeType(args[0]							,		0							,		0							,		*args[1:]							)
					elif hasattr(UpperCAmelCase__							,		"""co_kwonlyargcount"""							):
							if len(UpperCAmelCase__							) == 16:
									return CodeType(args[0]							,		*args[2:]							)
							elif len(UpperCAmelCase__							) == 15:
									return CodeType(*UpperCAmelCase__							)
							return CodeType(args[0]							,		0							,		*args[1:]							)
					if len(UpperCAmelCase__							) == 16:
							return CodeType(args[0]							,		*args[3:]							)
					elif len(UpperCAmelCase__							) == 15:
							return CodeType(args[0]							,		*args[2:]							)
					return CodeType(*UpperCAmelCase__							)

			def create_ipython_func(UpperCAmelCase__	:		int							,		UpperCAmelCase__	:		int							):
					def func():
							return returned_obj

					_a						: str    =			func.__code__
					# Use _create_code from dill in order to make it work for different python versions
					if datasets.config.PY_VERSION < version.parse("""3.8"""							):
							_a						: Any    =			_create_code_pyaa(
							    *[getattr(UpperCAmelCase__							,		UpperCAmelCase__							) if k != """co_filename""" else co_filename for k in code_args_pyaa]							)
					else:
							_a						: Tuple    =			code.replace(co_filename=UpperCAmelCase__							)
					return FunctionType(UpperCAmelCase__							,		func.__globals__							,		func.__name__							,		func.__defaults__							,		func.__closure__							)

			_a						: Union[str, Any]    =			"""<ipython-input-2-e0383a102aae>""", [0]
			_a						: Dict    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			_a						: Dict    =			"""<ipython-input-2-e0383a102aae>""", [1]
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			_a						: str    =			"""<ipython-input-5-713f6613acf3>""", [0]
			_a						: int    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)

			_a						: Optional[Any]    =			os.path.join(gettempdir()							,		"""ipykernel_12345"""							,		"""321456789.py"""							), [0]
			_a						: Dict    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			_a						: Tuple    =			os.path.join(gettempdir()							,		"""ipykernel_12345"""							,		"""321456789.py"""							), [1]
			_a						: Optional[int]    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			_a						: List[str]    =			os.path.join(gettempdir()							,		"""ipykernel_12345"""							,		"""654123987.py"""							), [0]
			_a						: str    =			mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__							,		UpperCAmelCase__							)							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







	def _lowercase			(  self	:		List[Any]							)      -> Optional[int]:
			_a						: Any    =			[0], [1]

			def func():
					return foo, bar

			_a						: Optional[int]    =			"""__main__"""

			def globalvars_mocka_side_effect(UpperCAmelCase__	:		str							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		int							):
					return {"foo": foo, "bar": bar}

			def globalvars_mocka_side_effect(UpperCAmelCase__	:		Optional[Any]							,		*UpperCAmelCase__	:		Dict							,		**UpperCAmelCase__	:		Optional[int]							):
					return {"bar": bar, "foo": foo}

			with patch("""dill.detect.globalvars"""							,		side_effect=UpperCAmelCase__							) as globalvars_mocka:
					_a						: Tuple    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
					self.assertGreater(globalvars_mocka.call_count							,		0							)
			with patch("""dill.detect.globalvars"""							,		side_effect=UpperCAmelCase__							) as globalvars_mocka:
					_a						: Dict    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
					self.assertGreater(globalvars_mocka.call_count							,		0							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







class  UpperCamelCase    (			snake_case_   ):


	def _lowercase			(  self	:		List[str]							)      -> List[str]:
			_a						: Any    =			Hasher.hash("""hello"""							)
			_a						: List[Any]    =			Hasher.hash("""hello"""							)
			_a						: Union[str, Any]    =			Hasher.hash("""there"""							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		Union[str, Any]							)      -> List[Any]:
			_a						: Any    =			Hasher.hash(Foo("""hello"""							)							)
			_a						: Optional[int]    =			Hasher.hash(Foo("""hello"""							)							)
			_a						: List[Any]    =			Hasher.hash(Foo("""there"""							)							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		int							)      -> List[str]:
			_a						: Optional[int]    =			Hasher()
			for x in ["hello", Foo("""hello"""							)]:
					hasher.update(UpperCAmelCase__							)
			_a						: Dict    =			hasher.hexdigest()
			_a						: Any    =			Hasher()
			for x in ["hello", Foo("""hello"""							)]:
					hasher.update(UpperCAmelCase__							)
			_a						: Any    =			hasher.hexdigest()
			_a						: Dict    =			Hasher()
			for x in ["there", Foo("""there"""							)]:
					hasher.update(UpperCAmelCase__							)
			_a						: Any    =			hasher.hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	def _lowercase			(  self	:		Optional[Any]							)      -> Optional[Any]:
			with self.assertRaises(pickle.PicklingError							):
					Hasher.hash(UnpicklableCallable(Foo("""hello"""							)							)							)


	def _lowercase			(  self	:		Union[str, Any]							)      -> str:
			_a						: Any    =			"""abc"""
			_a						: List[str]    =			[string, string]  # two strings have the same ids
			_a						: Optional[Any]    =			[string, string]
			_a						: Tuple    =			json.loads(f"""[\"{string}\", \"{string}\"]"""							)  # two strings have different ids
			self.assertIs(obja[0]							,		UpperCAmelCase__							)
			self.assertIs(obja[0]							,		obja[1]							)
			self.assertIs(obja[0]							,		UpperCAmelCase__							)
			self.assertIs(obja[0]							,		obja[1]							)
			self.assertIsNot(obja[0]							,		UpperCAmelCase__							)
			self.assertIsNot(obja[0]							,		obja[1]							)
			_a						: Any    =			Hasher.hash(UpperCAmelCase__							)
			_a						: Dict    =			Hasher.hash(UpperCAmelCase__							)
			_a						: List[str]    =			Hasher.hash(UpperCAmelCase__							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	@require_tiktoken
	def _lowercase			(  self	:		Union[str, Any]							)      -> Optional[int]:
			import tiktoken

			_a						: Union[str, Any]    =			tiktoken.get_encoding("""gpt2"""							)
			_a						: List[str]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Union[str, Any]    =			tiktoken.get_encoding("""r50k_base"""							)
			_a						: int    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Tuple    =			tiktoken.get_encoding("""gpt2"""							)
			_a						: Union[str, Any]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)


	@require_torch
	def _lowercase			(  self	:		List[Any]							)      -> Optional[int]:
			import torch

			_a						: Union[str, Any]    =			torch.tensor([1.0]							)
			_a						: Optional[int]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Dict    =			torch.tensor([2.0]							)
			_a						: Dict    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: List[Any]    =			torch.tensor([1.0]							)
			_a						: Optional[Any]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







	@require_spacy
	@require_spacy_model("""en_core_web_sm"""							)
	@require_spacy_model("""fr_core_news_sm"""							)
	@pytest.mark.integration
	def _lowercase			(  self	:		Any							)      -> Optional[Any]:
			import spacy

			_a						: Union[str, Any]    =			spacy.load("""en_core_web_sm"""							)
			_a						: Union[str, Any]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: int    =			spacy.load("""fr_core_news_sm"""							)
			_a						: Any    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			_a						: Dict    =			spacy.load("""en_core_web_sm"""							)
			_a						: List[str]    =			mda(datasets.utils.py_utils.dumps(UpperCAmelCase__							)							).hexdigest()
			self.assertEqual(UpperCAmelCase__							,		UpperCAmelCase__							)
			self.assertNotEqual(UpperCAmelCase__							,		UpperCAmelCase__							)







@pytest.mark.integration
def     lowerCAmelCase__  (  UpperCamelCase__       ):



		'''simple docstring'''
		_a						: Optional[Any]    =			tmp_path / """dir1"""
		_a						: Optional[Any]    =			tmp_path / """dir2"""
		dira.mkdir()
		dira.mkdir()
		_a						: int    =			"""script.py"""
		_a						: Tuple    =			dedent(
		    """
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    """       )
		_a						: List[str]    =			dira / script_filename
		_a						: int    =			dira / script_filename
		with script_patha.open("""w"""       ) as f:
				f.write(UpperCamelCase__       )
		with script_patha.open("""w"""       ) as f:
				f.write(UpperCamelCase__       )
		_a						: Optional[Any]    =			subprocess.check_output(["""python""", str(UpperCamelCase__       )]       )
		_a						: Union[str, Any]    =			subprocess.check_output(["""python""", str(UpperCamelCase__       )]       )
		assert fingerprinta == fingerprinta

def     lowerCAmelCase__  (  ):



		'''simple docstring'''
		_a						: int    =			{"""a""": [0, 1, 2]}
		_a						: Tuple    =			DatasetChild(InMemoryTable.from_pydict(UpperCamelCase__       )       )
		_a						: int    =			dataset.funca()._fingerprint
		assert expected_fingerprint == dataset.funca()._fingerprint
		assert expected_fingerprint != dataset.funca()._fingerprint

		with Pool(2       ) as p:
				assert expected_fingerprint == p.apply_async(dataset.funca       ).get()._fingerprint
				assert expected_fingerprint != p.apply_async(dataset.funca       ).get()._fingerprint

def     lowerCAmelCase__  (  ):



		'''simple docstring'''
		_a						: Tuple    =			{"""a""": [0, 1, 2]}






		class  UpperCamelCase    (			datasets.Dataset   ):


			@fingerprint_transform(inplace=UpperCAmelCase__							)
			def _lowercase			(  self	:		str							,		UpperCAmelCase__	:		Tuple							)      -> Any:
					return DummyDatasetChild(self.data							,		fingerprint=UpperCAmelCase__							)







		_a						: Any    =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__       )       ).func()






		class  UpperCamelCase    (			datasets.Dataset   ):


			@fingerprint_transform(inplace=UpperCAmelCase__							,		version="""1.0.0"""							)
			def _lowercase			(  self	:		Optional[int]							,		UpperCAmelCase__	:		Optional[int]							)      -> Dict:
					return DummyDatasetChild(self.data							,		fingerprint=UpperCAmelCase__							)







		_a						: int    =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__       )       ).func()






		class  UpperCamelCase    (			datasets.Dataset   ):


			@fingerprint_transform(inplace=UpperCAmelCase__							,		version="""2.0.0"""							)
			def _lowercase			(  self	:		int							,		UpperCAmelCase__	:		List[str]							)      -> str:
					return DummyDatasetChild(self.data							,		fingerprint=UpperCAmelCase__							)







		_a						: Tuple    =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__       )       ).func()

		assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}       ) == 3



def     lowerCAmelCase__  (  ):



		'''simple docstring'''
		_a						: Dict    =			Hasher()
		hasher.update(lambda UpperCamelCase__       : x       )









"""simple docstring"""




import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
 from .tokenization_llama import LlamaTokenizer
else:
 _snake_case					   =     None

_snake_case					   =     logging.get_logger(__name__)
_snake_case					   =     {'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

_snake_case   ,				_snake_case					   =     '[INST]', '[/INST]'
_snake_case   ,				_snake_case					   =     '<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
_snake_case					   =     'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'






class  UpperCamelCase    (			snake_case_   ):
 UpperCamelCase      : Any    =      VOCAB_FILES_NAMES
 UpperCamelCase      : Tuple    =      LlamaTokenizer
 UpperCamelCase      : Union[str, Any]    =      '''left'''
 UpperCamelCase      : str    =      ['''input_ids''', '''attention_mask''']


 def __init__(  self	:		Union[str, Any]							,		UpperCAmelCase__	:		Optional[int]=None							,		UpperCAmelCase__	:		Dict=None							,		UpperCAmelCase__	:		Optional[int]=False							,		UpperCAmelCase__	:		Optional[Any]="<unk>"							,		UpperCAmelCase__	:		str="<s>"							,		UpperCAmelCase__	:		List[Any]="</s>"							,		UpperCAmelCase__	:		Tuple=True							,		UpperCAmelCase__	:		int=False							,		**UpperCAmelCase__	:		Tuple							,		)      -> Dict:
   super().__init__(
       vocab_file=UpperCAmelCase__							,		tokenizer_file=UpperCAmelCase__							,		clean_up_tokenization_spaces=UpperCAmelCase__							,		unk_token=UpperCAmelCase__							,		bos_token=UpperCAmelCase__							,		eos_token=UpperCAmelCase__							,		**UpperCAmelCase__							,		)
   _a						: List[str]    =			add_bos_token
   _a						: Union[str, Any]    =			add_eos_token
   self.update_post_processor()

   _a						: List[str]    =			vocab_file
   _a						: Union[str, Any]    =			False if not self.vocab_file else True


 def _lowercase			(  self	:		List[str]							)      -> int:
   _a						: Dict    =			self.bos_token
   _a						: Union[str, Any]    =			self.bos_token_id

   _a						: int    =			self.eos_token
   _a						: str    =			self.eos_token_id

   _a						: Optional[Any]    =			f"""{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"""
   _a						: str    =			f"""{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"""

   _a						: Optional[int]    =			[]
   if self.add_bos_token:
     special_tokens.append((bos, bos_token_id)							)
   if self.add_eos_token:
     special_tokens.append((eos, eos_token_id)							)
   _a						: str    =			processors.TemplateProcessing(
       single=UpperCAmelCase__							,		pair=UpperCAmelCase__							,		special_tokens=UpperCAmelCase__							)


 @property
 def _lowercase			(  self	:		Optional[int]							)      -> Optional[Any]:
   return self._add_eos_token


 @property
 def _lowercase			(  self	:		Dict							)      -> str:
   return self._add_bos_token


 @add_eos_token.setter
 def _lowercase			(  self	:		Any							,		UpperCAmelCase__	:		int							)      -> List[str]:
   _a						: Any    =			value
   self.update_post_processor()


 @add_bos_token.setter
 def _lowercase			(  self	:		Any							,		UpperCAmelCase__	:		int							)      -> List[str]:
   _a						: Optional[Any]    =			value
   self.update_post_processor()


 def _lowercase			(  self	:		Dict							,		UpperCAmelCase__	:		str							,		UpperCAmelCase__	:		Optional[str] = None							)      -> Tuple[str]:
   if not self.can_save_slow_tokenizer:
     raise ValueError(
         """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
         """tokenizer."""							)

   if not os.path.isdir(UpperCAmelCase__							):
     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
     return
   _a						: str    =			os.path.join(
       UpperCAmelCase__							,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

   if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase__							):
     copyfile(self.vocab_file							,		UpperCAmelCase__							)

   return (out_vocab_file,)







 def _lowercase			(  self	:		Tuple							,		UpperCAmelCase__	:		"Conversation"							)      -> Optional[Any]:
   _a						: str    =			list(conversation.iter_texts()							)
   if not all([is_user for is_user, msg in dialogue[::2]]							) or not all(
       [not is_user for is_user, msg in dialogue[1::2]]							):
     raise ValueError(
         """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"""							)

   _a						: Tuple    =			[]
   if len(conversation.past_user_inputs							) > 0:
     if not conversation.past_user_inputs[0].startswith(UpperCAmelCase__							) or E_SYS not in conversation.past_user_inputs[0]:
       _a						: Union[str, Any]    =			(
           B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
       )
   elif not dialogue[0][1].startswith(UpperCAmelCase__							) or E_SYS not in dialogue[0][1]:
     _a						: str    =			(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

   dialog_tokens += sum(
       [
           [self.bos_token_id]
           + self.encode(
               f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """							,		add_special_tokens=UpperCAmelCase__							)
           + [self.eos_token_id]
           for prompt, answer in zip(dialogue[::2]							,		dialogue[1::2]							)
       ]							,		[]							,		)
   if not (dialogue[-1][0]):
     raise ValueError(f"""Last message must be from user, got {dialogue[-1]['role']}"""							)
   dialog_tokens += [self.bos_token_id] + self.encode(
       f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""							,		add_special_tokens=UpperCAmelCase__							)
   return dialog_tokens

