


import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

A  :			Optional[int]     				=				logging.get_logger(__name__)

A  :			List[Any]     				=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

A  :			int     				=				{
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

A  :			Optional[Any]     				=				{
    'microsoft/deberta-base': 5_1_2,
    'microsoft/deberta-large': 5_1_2,
    'microsoft/deberta-xlarge': 5_1_2,
    'microsoft/deberta-base-mnli': 5_1_2,
    'microsoft/deberta-large-mnli': 5_1_2,
    'microsoft/deberta-xlarge-mnli': 5_1_2,
}

A  :			str     				=				{
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}


def 					__lowerCAmelCase    (    ) ->						Optional[Any]:
				__a    							=					(
				    list(range(ord('''!'''						)		,					ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)		,					ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)		,					ord('''ÿ'''						) + 1						)						)
				)
				__a    							=					bs[:]
				__a    							=					0
				for b in range(2**8						):
								if b not in bs:
												bs.append(a__						)
												cs.append(2**8 + n						)
												n += 1
				__a    							=					[chr(a__						) for n in cs]
				return dict(zip(a__		,					a__						)						)


def 					__lowerCAmelCase    (    a__						) ->						Any:
				__a    							=					set()
				__a    							=					word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)						)
								__a    							=					char
				return pairs







class     __A(				a    ):
				snake_case_		     =					VOCAB_FILES_NAMES
				snake_case_		     =					PRETRAINED_VOCAB_FILES_MAP
				snake_case_		     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				snake_case_		     =					['''input_ids''', '''attention_mask''', '''token_type_ids''']




				def __init__(  self	,  _snake_case	,  _snake_case	,  _snake_case="replace"	,  _snake_case="[CLS]"	,  _snake_case="[SEP]"	,  _snake_case="[SEP]"	,  _snake_case="[CLS]"	,  _snake_case="[UNK]"	,  _snake_case="[PAD]"	,  _snake_case="[MASK]"	,  _snake_case=False	,  _snake_case=False	,  **_snake_case	,  )					->   Tuple:


								'''simple docstring'''




								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else bos_token
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else eos_token
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else sep_token
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else cls_token
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else unk_token
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								__a    							=					AddedToken(_snake_case	,  lstrip=_snake_case	,  rstrip=_snake_case       ) if isinstance(_snake_case	,  _snake_case       ) else mask_token

								super().__init__(
								    errors=_snake_case	,  bos_token=_snake_case	,  eos_token=_snake_case	,  unk_token=_snake_case	,  sep_token=_snake_case	,  cls_token=_snake_case	,  pad_token=_snake_case	,  mask_token=_snake_case	,  add_prefix_space=_snake_case	,  add_bos_token=_snake_case	,  **_snake_case	,  )
								__a    							=					add_bos_token

								with open(_snake_case	,  encoding='''utf-8'''       ) as vocab_handle:
												__a    							=					json.load(_snake_case       )
								__a    							=					{v: k for k, v in self.encoder.items()}
								__a    							=					errors  # how to handle errors in decoding
								__a    							=					bytes_to_unicode()
								__a    							=					{v: k for k, v in self.byte_encoder.items()}
								with open(_snake_case	,  encoding='''utf-8'''       ) as merges_handle:
												__a    							=					merges_handle.read().split('''\n'''       )[1:-1]
								__a    							=					[tuple(merge.split()       ) for merge in bpe_merges]
								__a    							=					dict(zip(_snake_case	,  range(len(_snake_case       )       )       )       )
								__a    							=					{}
								__a    							=					add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								__a    							=					re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )




				@property
				# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
				def        SCREAMING_SNAKE_CASE_	(  self       )					->   List[str]:


								'''simple docstring'''




								return len(self.encoder       )




				def        SCREAMING_SNAKE_CASE_	(  self       )					->   Tuple:


								'''simple docstring'''




								return dict(self.encoder	,  **self.added_tokens_encoder       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Optional[Any]:


								'''simple docstring'''




								if token in self.cache:
												return self.cache[token]
								__a    							=					tuple(_snake_case       )
								__a    							=					get_pairs(_snake_case       )

								if not pairs:
												return token

								while True:
												__a    							=					min(_snake_case	,  key=lambda _snake_case       : self.bpe_ranks.get(_snake_case	,  float('''inf'''       )       )       )
												if bigram not in self.bpe_ranks:
																break
												__a							,  __a    							=					bigram
												__a    							=					[]
												__a    							=					0
												while i < len(_snake_case       ):
																try:
																				__a    							=					word.index(_snake_case	,  _snake_case       )
																except ValueError:
																				new_word.extend(word[i:]       )
																				break
																else:
																				new_word.extend(word[i:j]       )
																				__a    							=					j

																if word[i] == first and i < len(_snake_case       ) - 1 and word[i + 1] == second:
																				new_word.append(first + second       )
																				i += 2
																else:
																				new_word.append(word[i]       )
																				i += 1
												__a    							=					tuple(_snake_case       )
												__a    							=					new_word
												if len(_snake_case       ) == 1:
																break
												else:
																__a    							=					get_pairs(_snake_case       )
								__a    							=					''' '''.join(_snake_case       )
								__a    							=					word
								return word




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case = None       )					->   List[int]:


								'''simple docstring'''




								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								__a    							=					[self.cls_token_id]
								__a    							=					[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case = None	,  _snake_case = False       )					->   List[int]:


								'''simple docstring'''




								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_snake_case	,  token_ids_a=_snake_case	,  already_has_special_tokens=_snake_case       )

								if token_ids_a is None:
												return [1] + ([0] * len(_snake_case       )) + [1]
								return [1] + ([0] * len(_snake_case       )) + [1] + ([0] * len(_snake_case       )) + [1]




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case = None       )					->   List[int]:


								'''simple docstring'''




								__a    							=					[self.sep_token_id]
								__a    							=					[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
								return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   int:


								'''simple docstring'''




								__a    							=					[]
								for token in re.findall(self.pat	,  _snake_case       ):
												__a    							=					''''''.join(
												    self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case       ).split(''' '''       )       )
								return bpe_tokens




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Optional[int]:


								'''simple docstring'''




								return self.encoder.get(_snake_case	,  self.encoder.get(self.unk_token       )       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tuple:


								'''simple docstring'''




								return self.decoder.get(_snake_case       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Optional[Any]:


								'''simple docstring'''




								__a    							=					''''''.join(_snake_case       )
								__a    							=					bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''	,  errors=self.errors       )
								return text




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case = None       )					->   Tuple[str]:


								'''simple docstring'''




								if not os.path.isdir(_snake_case       ):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
								__a    							=					os.path.join(
								    _snake_case	,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
								__a    							=					os.path.join(
								    _snake_case	,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

								with open(_snake_case	,  '''w'''	,  encoding='''utf-8'''       ) as f:
												f.write(json.dumps(self.encoder	,  indent=2	,  sort_keys=_snake_case	,  ensure_ascii=_snake_case       ) + '''\n'''       )

								__a    							=					0
								with open(_snake_case	,  '''w'''	,  encoding='''utf-8'''       ) as writer:
												writer.write('''#version: 0.2\n'''       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,  key=lambda _snake_case       : kv[1]       ):
																if index != token_index:
																				logger.warning(
																				    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																				    ''' Please check that the tokenizer is not corrupted!'''       )
																				__a    							=					token_index
																writer.write(''' '''.join(_snake_case       ) + '''\n'''       )
																index += 1

								return vocab_file, merge_file




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case=False	,  **_snake_case       )					->   Optional[Any]:


								'''simple docstring'''




								__a    							=					kwargs.pop('''add_prefix_space'''	,  self.add_prefix_space       )
								if (is_split_into_words or add_prefix_space) and (len(_snake_case       ) > 0 and not text[0].isspace()):
												__a    							=					''' ''' + text
								return (text, kwargs)




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   List[int]:


								'''simple docstring'''




								__a    							=					[]
								for is_user, text in conversation.iter_texts():
												input_ids.extend(self.encode(_snake_case	,  add_special_tokens=_snake_case       ) + [self.eos_token_id]       )
								if len(_snake_case       ) > self.model_max_length:
												__a    							=					input_ids[-self.model_max_length :]
								return input_ids

import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


A  :			List[str]     				=				logging.get_logger(__name__)







class     __A(				nn.Module    ):




				def __init__(  self       )					->   Union[str, Any]:


								'''simple docstring'''




								super().__init__()
								if version.parse(torch.__version__       ) < version.parse('''1.12.0'''       ):
												raise ImportError(
												    F"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
												    '''PytorchGELUTanh. Please upgrade torch.'''       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return nn.functional.gelu(_snake_case	,  approximate='''tanh'''       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi       ) * (input + 0.04_4715 * torch.pow(_snake_case	,  3.0       ))       ))







class     __A(				nn.Module    ):




				def __init__(  self	,  _snake_case = False       )					->   Tuple:


								'''simple docstring'''




								super().__init__()
								if use_gelu_python:
												__a    							=					self._gelu_python
								else:
												__a    							=					nn.functional.gelu




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0       )       ))




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return self.act(_snake_case       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * torch.sigmoid(1.702 * input       )







class     __A(				nn.Module    ):




				def __init__(  self	,  _snake_case	,  _snake_case       )					->   Optional[int]:


								'''simple docstring'''




								if min > max:
												raise ValueError(F"""min should be < max (got min: {min}, max: {max})"""       )

								super().__init__()
								__a    							=					min
								__a    							=					max




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return torch.clip(gelu(_snake_case       )	,  self.min	,  self.max       )







class     __A(				nn.Module    ):




				def __init__(  self       )					->   str:


								'''simple docstring'''




								super().__init__()
								__a    							=					math.sqrt(2 / math.pi       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(_snake_case	,  3       ))       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return nn.functional.silu(_snake_case       )







class     __A(				nn.Module    ):




				def __init__(  self       )					->   List[Any]:


								'''simple docstring'''




								super().__init__()
								if version.parse(torch.__version__       ) < version.parse('''1.9.0'''       ):
												__a    							=					self._mish_python
								else:
												__a    							=					nn.functional.mish




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * torch.tanh(nn.functional.softplus(_snake_case       )       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return self.act(_snake_case       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case=0.70_7107	,  _snake_case=0.28_2095       )					->   int:


								'''simple docstring'''




								__a    							=					(input - mu).div(sigma * math.sqrt(2.0       )       )
								return 0.5 * (1.0 + torch.erf(_snake_case       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Any:


								'''simple docstring'''




								__a    							=					nn.functional.relu(_snake_case       )
								__a    							=					torch.square(_snake_case       )
								return squared







class     __A(				a    ):




				def __getitem__(  self	,  _snake_case       )					->   Dict:


								'''simple docstring'''




								__a    							=					super().__getitem__(_snake_case       )
								__a							,  __a    							=					content if isinstance(_snake_case	,  _snake_case       ) else (content, {})
								return cls(**_snake_case       )



A  :			str     				=				{
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -1_0, 'max': 1_0}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
A  :			Any     				=				ClassInstantier(ACTaCLS)


def 					__lowerCAmelCase    (    a__						) ->						Optional[int]:
				if activation_string in ACTaFN:
								return ACTaFN[activation_string]
				else:
								raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}"""						)


# For backwards compatibility with: from activations import gelu_python
A  :			Optional[Any]     				=				get_activation('gelu_python')
A  :			str     				=				get_activation('gelu_new')
A  :			Any     				=				get_activation('gelu')
A  :			int     				=				get_activation('gelu_fast')
A  :			Any     				=				get_activation('quick_gelu')
A  :			int     				=				get_activation('silu')
A  :			List[str]     				=				get_activation('mish')






A  :			Dict     				=				get_activation('linear')