






"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE      =      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      =      {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__SCREAMING_SNAKE_CASE      =      {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

__SCREAMING_SNAKE_CASE      =      {
    'roberta-base': 5_12,
    'roberta-large': 5_12,
    'roberta-large-mnli': 5_12,
    'distilroberta-base': 5_12,
    'roberta-base-openai-detector': 5_12,
    'roberta-large-openai-detector': 5_12,
}


@lru_cache()
def 					UpperCAmelCase			(				):





						'''simple docstring'''






						lowerCAmelCase      :Dict				  = (
						    list(range(ord('!'   )    ,			ord('~'   ) + 1   )   ) + list(range(ord('¡'   )    ,			ord('¬'   ) + 1   )   ) + list(range(ord('®'   )    ,			ord('ÿ'   ) + 1   )   )
						)
						lowerCAmelCase      :Optional[int]				  = bs[:]
						lowerCAmelCase      :List[Any]				  = 0
						for b in range(2**8   ):
												if b not in bs:
																		bs.append(a__   )
																		cs.append(2**8 + n   )
																		n += 1
						lowerCAmelCase      :int				  = [chr(a__   ) for n in cs]
						return dict(zip(a__    ,			a__   )   )


def 					UpperCAmelCase			(				a__   ):





						'''simple docstring'''






						lowerCAmelCase      :Tuple				  = set()
						lowerCAmelCase      :Optional[int]				  = word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)   )
												lowerCAmelCase      :Union[str, Any]				  = char
						return pairs







class 	__UpperCamelCase    (    UpperCamelCase	):
					lowercase_						:							int			=     VOCAB_FILES_NAMES
					lowercase_						:							Any			=     PRETRAINED_VOCAB_FILES_MAP
					lowercase_						:							Optional[Any]			=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_						:							Optional[int]			=     ["""input_ids""", """attention_mask"""]
					def __init__(       self       :  Tuple    ,   UpperCAmelCase       :  Tuple    ,   UpperCAmelCase       :  Dict    ,   UpperCAmelCase       :  int="replace"    ,   UpperCAmelCase       :  Any="<s>"    ,   UpperCAmelCase       :  List[str]="</s>"    ,   UpperCAmelCase       :  Any="</s>"    ,   UpperCAmelCase       :  Optional[int]="<s>"    ,   UpperCAmelCase       :  Optional[int]="<unk>"    ,   UpperCAmelCase       :  Optional[Any]="<pad>"    ,   UpperCAmelCase       :  Tuple="<mask>"    ,   UpperCAmelCase       :  List[Any]=False    ,   **UpperCAmelCase       :  List[str]    ,   )  ->					List[Any]:
											lowerCAmelCase      :str				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else bos_token
											lowerCAmelCase      :Tuple				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else eos_token
											lowerCAmelCase      :List[str]				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else sep_token
											lowerCAmelCase      :List[str]				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else cls_token
											lowerCAmelCase      :str				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else unk_token
											lowerCAmelCase      :Dict				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											lowerCAmelCase      :int				  = AddedToken(UpperCAmelCase    ,   lstrip=UpperCAmelCase    ,   rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,   UpperCAmelCase      ) else mask_token

											super().__init__(
											    errors=UpperCAmelCase    ,   bos_token=UpperCAmelCase    ,   eos_token=UpperCAmelCase    ,   unk_token=UpperCAmelCase    ,   sep_token=UpperCAmelCase    ,   cls_token=UpperCAmelCase    ,   pad_token=UpperCAmelCase    ,   mask_token=UpperCAmelCase    ,   add_prefix_space=UpperCAmelCase    ,   **UpperCAmelCase    ,   )

											with open(UpperCAmelCase    ,   encoding='utf-8'      ) as vocab_handle:
																	lowerCAmelCase      :Union[str, Any]				  = json.load(UpperCAmelCase      )
											lowerCAmelCase      :List[Any]				  = {v: k for k, v in self.encoder.items()}
											lowerCAmelCase      :Optional[Any]				  = errors  # how to handle errors in decoding
											lowerCAmelCase      :Optional[int]				  = bytes_to_unicode()
											lowerCAmelCase      :List[Any]				  = {v: k for k, v in self.byte_encoder.items()}
											with open(UpperCAmelCase    ,   encoding='utf-8'      ) as merges_handle:
																	lowerCAmelCase      :Union[str, Any]				  = merges_handle.read().split('\n'      )[1:-1]
											lowerCAmelCase      :str				  = [tuple(merge.split()      ) for merge in bpe_merges]
											lowerCAmelCase      :List[str]				  = dict(zip(UpperCAmelCase    ,   range(len(UpperCAmelCase      )      )      )      )
											lowerCAmelCase      :str				  = {}
											lowerCAmelCase      :Tuple				  = add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											lowerCAmelCase      :Optional[Any]				  = re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )
					@property
					def 					UpperCAmelCase__   (       self       :  Optional[int]      )  ->					str:
											return len(self.encoder      )
					def 					UpperCAmelCase__   (       self       :  Tuple      )  ->					Optional[int]:
											return dict(self.encoder    ,   **self.added_tokens_encoder      )
					def 					UpperCAmelCase__   (       self       :  Optional[Any]    ,   UpperCAmelCase       :  Optional[int]      )  ->					List[Any]:
											if token in self.cache:
																	return self.cache[token]
											lowerCAmelCase      :Dict				  = tuple(UpperCAmelCase      )
											lowerCAmelCase      :str				  = get_pairs(UpperCAmelCase      )

											if not pairs:
																	return token

											while True:
																	lowerCAmelCase      :Tuple				  = min(UpperCAmelCase    ,   key=lambda UpperCAmelCase      : self.bpe_ranks.get(UpperCAmelCase    ,   float('inf'      )      )      )
																	if bigram not in self.bpe_ranks:
																							break
																	lowerCAmelCase  ,   lowerCAmelCase      :List[Any]				  = bigram
																	lowerCAmelCase      :List[Any]				  = []
																	lowerCAmelCase      :Dict				  = 0
																	while i < len(UpperCAmelCase      ):
																							try:
																													lowerCAmelCase      :Optional[Any]				  = word.index(UpperCAmelCase    ,   UpperCAmelCase      )
																							except ValueError:
																													new_word.extend(word[i:]      )
																													break
																							else:
																													new_word.extend(word[i:j]      )
																													lowerCAmelCase      :Dict				  = j

																							if word[i] == first and i < len(UpperCAmelCase      ) - 1 and word[i + 1] == second:
																													new_word.append(first + second      )
																													i += 2
																							else:
																													new_word.append(word[i]      )
																													i += 1
																	lowerCAmelCase      :List[Any]				  = tuple(UpperCAmelCase      )
																	lowerCAmelCase      :List[str]				  = new_word
																	if len(UpperCAmelCase      ) == 1:
																							break
																	else:
																							lowerCAmelCase      :List[Any]				  = get_pairs(UpperCAmelCase      )
											lowerCAmelCase      :Tuple				  = ' '.join(UpperCAmelCase      )
											lowerCAmelCase      :str				  = word
											return word
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  Dict      )  ->					int:
											lowerCAmelCase      :List[Any]				  = []
											for token in re.findall(self.pat    ,   UpperCAmelCase      ):
																	lowerCAmelCase      :List[Any]				  = ''.join(
																	    self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase      ).split(' '      )      )
											return bpe_tokens
					def 					UpperCAmelCase__   (       self       :  str    ,   UpperCAmelCase       :  List[str]      )  ->					int:
											return self.encoder.get(UpperCAmelCase    ,   self.encoder.get(self.unk_token      )      )
					def 					UpperCAmelCase__   (       self       :  List[str]    ,   UpperCAmelCase       :  Union[str, Any]      )  ->					Dict:
											return self.decoder.get(UpperCAmelCase      )
					def 					UpperCAmelCase__   (       self       :  Tuple    ,   UpperCAmelCase       :  Optional[int]      )  ->					List[str]:
											lowerCAmelCase      :int				  = ''.join(UpperCAmelCase      )
											lowerCAmelCase      :List[Any]				  = bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'    ,   errors=self.errors      )
											return text
					def 					UpperCAmelCase__   (       self       :  List[str]    ,   UpperCAmelCase       :  str    ,   UpperCAmelCase       :  Optional[str] = None      )  ->					Tuple[str]:
											if not os.path.isdir(UpperCAmelCase      ):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""      )
																	return
											lowerCAmelCase      :Dict				  = os.path.join(
											    UpperCAmelCase    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
											lowerCAmelCase      :Dict				  = os.path.join(
											    UpperCAmelCase    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

											with open(UpperCAmelCase    ,   'w'    ,   encoding='utf-8'      ) as f:
																	f.write(json.dumps(self.encoder    ,   indent=2    ,   sort_keys=UpperCAmelCase    ,   ensure_ascii=UpperCAmelCase      ) + '\n'      )

											lowerCAmelCase      :int				  = 0
											with open(UpperCAmelCase    ,   'w'    ,   encoding='utf-8'      ) as writer:
																	writer.write('#version: 0.2\n'      )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,   key=lambda UpperCAmelCase      : kv[1]      ):
																							if index != token_index:
																													logger.warning(
																													    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																													    ' Please check that the tokenizer is not corrupted!'      )
																													lowerCAmelCase      :List[Any]				  = token_index
																							writer.write(' '.join(UpperCAmelCase      ) + '\n'      )
																							index += 1

											return vocab_file, merge_file
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  List[int]    ,   UpperCAmelCase       :  Optional[List[int]] = None      )  ->					List[int]:
											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											lowerCAmelCase      :Optional[int]				  = [self.cls_token_id]
											lowerCAmelCase      :Optional[Any]				  = [self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]    ,   UpperCAmelCase       :  List[int]    ,   UpperCAmelCase       :  Optional[List[int]] = None    ,   UpperCAmelCase       :  bool = False      )  ->					List[int]:
											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=UpperCAmelCase    ,   token_ids_a=UpperCAmelCase    ,   already_has_special_tokens=UpperCAmelCase      )

											if token_ids_a is None:
																	return [1] + ([0] * len(UpperCAmelCase      )) + [1]
											return [1] + ([0] * len(UpperCAmelCase      )) + [1, 1] + ([0] * len(UpperCAmelCase      )) + [1]
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  List[int]    ,   UpperCAmelCase       :  Optional[List[int]] = None      )  ->					List[int]:
											lowerCAmelCase      :Any				  = [self.sep_token_id]
											lowerCAmelCase      :Tuple				  = [self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep      ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]
					def 					UpperCAmelCase__   (       self       :  Tuple    ,   UpperCAmelCase       :  int    ,   UpperCAmelCase       :  Any=False    ,   **UpperCAmelCase       :  List[str]      )  ->					List[Any]:
											lowerCAmelCase      :Dict				  = kwargs.pop('add_prefix_space'    ,   self.add_prefix_space      )
											if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase      ) > 0 and not text[0].isspace()):
																	lowerCAmelCase      :List[Any]				  = ' ' + text
											return (text, kwargs)







"""simple docstring"""


import safs




class 	__UpperCamelCase    (    safs.SaFileSystem	):
					pass