


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case__     							= logging.get_logger(__name__)

snake_case__     							= {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

snake_case__     							= {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

snake_case__     							= {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}
@lru_cache()
def  __SCREAMING_SNAKE_CASE       (				)       ->  Any:
						__magic_name__				:							Optional[Any]				=      (
						    list(range(ord('!'				)      ,						ord('~'				) + 1				)				) + list(range(ord('¡'				)      ,						ord('¬'				) + 1				)				) + list(range(ord('®'				)      ,						ord('ÿ'				) + 1				)				)
						)
						__magic_name__				:							List[str]				=      bs[:]
						__magic_name__				:							int				=      0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(lowercase__				)
																		cs.append(2**8 + n				)
																		n += 1
						__magic_name__				:							List[str]				=      [chr(lowercase__				) for n in cs]
						return dict(zip(lowercase__      ,						lowercase__				)				)
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :List[Any]				)       ->  int:
						__magic_name__				:							Union[str, Any]				=      set()
						__magic_name__				:							List[Any]				=      word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												__magic_name__				:							Union[str, Any]				=      char
						return pairs



class     snake_case__  (      _A  ):
				__lowerCAmelCase		=      VOCAB_FILES_NAMES
				__lowerCAmelCase		=      PRETRAINED_VOCAB_FILES_MAP
				__lowerCAmelCase		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCAmelCase		=      ['''input_ids''', '''attention_mask''']
				def __init__(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase="replace"     ,      _UpperCAmelCase="<s>"     ,      _UpperCAmelCase="</s>"     ,      _UpperCAmelCase="</s>"     ,      _UpperCAmelCase="<s>"     ,      _UpperCAmelCase="<unk>"     ,      _UpperCAmelCase="<pad>"     ,      _UpperCAmelCase="<mask>"     ,      _UpperCAmelCase=False     ,      **_UpperCAmelCase     ,      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else bos_token
										__magic_name__				:							Union[str, Any]				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else eos_token
										__magic_name__				:							Any				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else sep_token
										__magic_name__				:							int				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else cls_token
										__magic_name__				:							int				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else unk_token
										__magic_name__				:							Union[str, Any]				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										__magic_name__				:							int				=      AddedToken(_UpperCAmelCase     ,      lstrip=_UpperCAmelCase     ,      rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ) else mask_token

										super().__init__(
										    errors=_UpperCAmelCase     ,      bos_token=_UpperCAmelCase     ,      eos_token=_UpperCAmelCase     ,      unk_token=_UpperCAmelCase     ,      sep_token=_UpperCAmelCase     ,      cls_token=_UpperCAmelCase     ,      pad_token=_UpperCAmelCase     ,      mask_token=_UpperCAmelCase     ,      add_prefix_space=_UpperCAmelCase     ,      **_UpperCAmelCase     ,      )

										with open(_UpperCAmelCase     ,      encoding='utf-8'      ) as vocab_handle:
																__magic_name__				:							Any				=      json.load(_UpperCAmelCase      )
										__magic_name__				:							Dict				=      {v: k for k, v in self.encoder.items()}
										__magic_name__				:							Tuple				=      errors  # how to handle errors in decoding
										__magic_name__				:							int				=      bytes_to_unicode()
										__magic_name__				:							List[Any]				=      {v: k for k, v in self.byte_encoder.items()}
										with open(_UpperCAmelCase     ,      encoding='utf-8'      ) as merges_handle:
																__magic_name__				:							str				=      merges_handle.read().split('\n'      )[1:-1]
										__magic_name__				:							Optional[Any]				=      [tuple(merge.split()      ) for merge in bpe_merges]
										__magic_name__				:							Tuple				=      dict(zip(_UpperCAmelCase     ,      range(len(_UpperCAmelCase      )      )      )      )
										__magic_name__				:							Union[str, Any]				=      {}
										__magic_name__				:							Tuple				=      add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										__magic_name__				:							List[Any]				=      re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )
				@property
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										return len(self.encoder      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										return dict(self.encoder     ,      **self.added_tokens_encoder      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										if token in self.cache:
																return self.cache[token]
										__magic_name__				:							List[str]				=      tuple(_UpperCAmelCase      )
										__magic_name__				:							Dict				=      get_pairs(_UpperCAmelCase      )

										if not pairs:
																return token

										while True:
																__magic_name__				:							Optional[Any]				=      min(_UpperCAmelCase     ,      key=lambda _UpperCAmelCase      : self.bpe_ranks.get(_UpperCAmelCase     ,      float('inf'      )      )      )
																if bigram not in self.bpe_ranks:
																						break
																__magic_name__		, __magic_name__				:							List[str]				=      bigram
																__magic_name__				:							Optional[int]				=      []
																__magic_name__				:							Dict				=      0
																while i < len(_UpperCAmelCase      ):
																						try:
																												__magic_name__				:							Optional[int]				=      word.index(_UpperCAmelCase     ,      _UpperCAmelCase      )
																						except ValueError:
																												new_word.extend(word[i:]      )
																												break
																						else:
																												new_word.extend(word[i:j]      )
																												__magic_name__				:							Union[str, Any]				=      j

																						if word[i] == first and i < len(_UpperCAmelCase      ) - 1 and word[i + 1] == second:
																												new_word.append(first + second      )
																												i += 2
																						else:
																												new_word.append(word[i]      )
																												i += 1
																__magic_name__				:							Dict				=      tuple(_UpperCAmelCase      )
																__magic_name__				:							Any				=      new_word
																if len(_UpperCAmelCase      ) == 1:
																						break
																else:
																						__magic_name__				:							Union[str, Any]				=      get_pairs(_UpperCAmelCase      )
										__magic_name__				:							List[Any]				=      ' '.join(_UpperCAmelCase      )
										__magic_name__				:							int				=      word
										return word
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							int				=      []
										for token in re.findall(self.pat     ,      _UpperCAmelCase      ):
																__magic_name__				:							int				=      ''.join(
																    self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase      ).split(' '      )      )
										return bpe_tokens
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										return self.encoder.get(_UpperCAmelCase     ,      self.encoder.get(self.unk_token      )      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										return self.decoder.get(_UpperCAmelCase      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Union[str, Any]				=      ''.join(_UpperCAmelCase      )
										__magic_name__				:							Tuple				=      bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'     ,      errors=self.errors      )
										return text
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase = None      ):




										"""simple docstring"""




										if not os.path.isdir(_UpperCAmelCase      ):
																logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
																return
										__magic_name__				:							Union[str, Any]				=      os.path.join(
										    _UpperCAmelCase     ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
										__magic_name__				:							List[str]				=      os.path.join(
										    _UpperCAmelCase     ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

										with open(_UpperCAmelCase     ,      'w'     ,      encoding='utf-8'      ) as f:
																f.write(json.dumps(self.encoder     ,      indent=2     ,      sort_keys=_UpperCAmelCase     ,      ensure_ascii=_UpperCAmelCase      ) + '\n'      )

										__magic_name__				:							Any				=      0
										with open(_UpperCAmelCase     ,      'w'     ,      encoding='utf-8'      ) as writer:
																writer.write('#version: 0.2\n'      )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,      key=lambda _UpperCAmelCase      : kv[1]      ):
																						if index != token_index:
																												logger.warning(
																												    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																												    ' Please check that the tokenizer is not corrupted!'      )
																												__magic_name__				:							Dict				=      token_index
																						writer.write(' '.join(_UpperCAmelCase      ) + '\n'      )
																						index += 1

										return vocab_file, merge_file
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase = None      ):




										"""simple docstring"""




										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										__magic_name__				:							Tuple				=      [self.cls_token_id]
										__magic_name__				:							Union[str, Any]				=      [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase = None     ,      _UpperCAmelCase = False      ):




										"""simple docstring"""




										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_UpperCAmelCase     ,      token_ids_a=_UpperCAmelCase     ,      already_has_special_tokens=_UpperCAmelCase      )

										if token_ids_a is None:
																return [1] + ([0] * len(_UpperCAmelCase      )) + [1]
										return [1] + ([0] * len(_UpperCAmelCase      )) + [1, 1] + ([0] * len(_UpperCAmelCase      )) + [1]
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase = None      ):




										"""simple docstring"""




										__magic_name__				:							Union[str, Any]				=      [self.sep_token_id]
										__magic_name__				:							List[Any]				=      [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep      ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]





				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase=False     ,      **_UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      kwargs.pop('add_prefix_space'     ,      self.add_prefix_space      )
										if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase      ) > 0 and not text[0].isspace()):
																__magic_name__				:							int				=      ' ' + text
										return (text, kwargs)






import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
			import torch


snake_case__     							= logging.get_logger(__name__)



class     snake_case__  :
				def __init__(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase=21     ,      _UpperCAmelCase=[]     ,      _UpperCAmelCase=None      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      model
										__magic_name__				:							List[str]				=      trace_batch_nums
										__magic_name__				:							int				=      abort_after_batch_num

										# keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
										__magic_name__				:							Union[str, Any]				=      collections.deque([]     ,      _UpperCAmelCase      )
										__magic_name__				:							Union[str, Any]				=      []
										__magic_name__				:							List[str]				=      0
										__magic_name__				:							Optional[int]				=      0
										__magic_name__				:							Any				=      False
										__magic_name__				:							str				=      '                 '

										self.analyse_model()

										self.register_forward_hook()
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase=None      ):




										"""simple docstring"""




										if frame is not None:
																self.expand_frame(_UpperCAmelCase      )
										self.frames.append('\n'.join(self.frame      )      )
										__magic_name__				:							List[Any]				=      []  # start a new frame
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										self.frame.append(_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										print('\n'.join(self.frames      )      )
										__magic_name__				:							List[str]				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							List[Any]				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										print(F"\nDetected inf/nan during batch_number={self.batch_number}"      )
										print(F"Last {len(self.frames      )} forward frames:"      )
										print(F"{'abs min':8} {'abs max':8} metadata"      )
										print('\n'.join(self.frames      )      )
										print('\n\n'      )
										__magic_name__				:							Any				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							int				=      {m: name for name, m in self.model.named_modules()}
										# self.longest_module_name = max(len(v) for v in self.module_names.values())
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										if torch.is_tensor(_UpperCAmelCase      ):
																self.expand_frame(get_abs_min_max(_UpperCAmelCase     ,      _UpperCAmelCase      )      )
																if detect_overflow(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																						__magic_name__				:							List[str]				=      True
										elif var is None:
																self.expand_frame(F"{'None':>17} {ctx}"      )
										else:
																self.expand_frame(F"{'not a tensor':>17} {ctx}"      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.expand_frame(F"\n\n{self.prefix} *** Starting batch number={self.batch_number} ***"      )
										self.expand_frame(F"{'abs min':8} {'abs max':8} metadata"      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.expand_frame(F"{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n"      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										self.expand_frame(F"{self.prefix} {self.module_names[module]} {module.__class__.__name__}"      )

										# params
										for name, p in module.named_parameters(recurse=_UpperCAmelCase      ):
																self.analyse_variable(_UpperCAmelCase     ,      _UpperCAmelCase      )

										# inputs
										if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																for i, x in enumerate(_UpperCAmelCase      ):
																						self.analyse_variable(_UpperCAmelCase     ,      F"input[{i}]"      )
										else:
																self.analyse_variable(_UpperCAmelCase     ,      'input'      )

										# outputs
										if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																for i, x in enumerate(_UpperCAmelCase      ):
																						# possibly a tuple of tuples
																						if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																												for j, y in enumerate(_UpperCAmelCase      ):
																																		self.analyse_variable(_UpperCAmelCase     ,      F"output[{i}][{j}]"      )
																						else:
																												self.analyse_variable(_UpperCAmelCase     ,      F"output[{i}]"      )
										else:
																self.analyse_variable(_UpperCAmelCase     ,      'output'      )

										self.save_frame()
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.model.apply(self._register_forward_hook      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										module.register_forward_hook(self.forward_hook      )





				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Optional[int]				=      False

										__magic_name__				:							Any				=      True if self.batch_number in self.trace_batch_nums else False
										if trace_mode:
																self.reset_saved_frames()

										if self.total_calls == 0:
																self.batch_start_frame()
										self.total_calls += 1

										# count batch numbers - the very first forward hook of the batch will be called when the
										# batch completes - i.e. it gets called very last - we know this batch has finished
										if module == self.model:
																self.batch_number += 1
																__magic_name__				:							str				=      True

										self.create_frame(_UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      )

										# if last_frame_of_batch:
										#     self.batch_end_frame()

										if trace_mode:
																self.trace_frames()

										if last_frame_of_batch:
																self.batch_start_frame()

										if self.detected_overflow and not trace_mode:
																self.dump_saved_frames()

																# now we can abort, as it's pointless to continue running
																raise ValueError(
																    'DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '
																    'Please scroll up above this traceback to see the activation values prior to this event.'      )

										# abort after certain batch if requested to do so
										if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
																raise ValueError(
																    F"DebugUnderflowOverflow: aborting after {self.batch_number} batches due to"
																    F" `abort_after_batch_num={self.abort_after_batch_num}` arg"      )






def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Union[str, Any]      ,						lowercase__  :Any				)       ->  Any:
						__magic_name__				:							List[Any]				=      var.abs()
						return f"{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}"
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple      ,						lowercase__  :Optional[int]				)       ->  Optional[Any]:
						__magic_name__				:							str				=      False
						if torch.isnan(lowercase__				).any().item():
												__magic_name__				:							Optional[Any]				=      True
												print(f"{ctx} has nans"				)
						if torch.isinf(lowercase__				).any().item():
												__magic_name__				:							str				=      True
												print(f"{ctx} has infs"				)

						# if needed to monitor large elements can enable the following
						if 0:  # and detected:
												__magic_name__				:							Any				=      var[torch.ge(var.abs()      ,						1_0_0				)]
												if naaa.numel() > 0:
																		print(f"{ctx}:  n100={naaa.numel()}"				)
												__magic_name__				:							Union[str, Any]				=      var[torch.ge(var.abs()      ,						1_0_0_0				)]
												if naaaa.numel() > 0:
																		print(f"{ctx}: n1000={naaaa.numel()}"				)
												__magic_name__				:							Any				=      var[torch.ge(var.abs()      ,						1_0_0_0_0				)]
												if naaaaa.numel() > 0:
																		print(f"{ctx}: n10000={naaaaa.numel()}"				)

						if 0:
												print(f"min={var.min():9.2e} max={var.max():9.2e}"				)

						if 0:
												print(f"min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})"				)

						return detected



class     snake_case__  (      _A  ):
				__lowerCAmelCase		=      '''underflow_overflow'''
				__lowerCAmelCase		=      '''tpu_metrics_debug'''


