




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE:    int											=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE:    int											=   {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

SCREAMING_SNAKE_CASE:    str											=   {
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
}

SCREAMING_SNAKE_CASE:    List[Any]											=   {
    '''roberta-base''': 5_1_2,
    '''roberta-large''': 5_1_2,
    '''roberta-large-mnli''': 5_1_2,
    '''distilroberta-base''': 5_1_2,
    '''roberta-base-openai-detector''': 5_1_2,
    '''roberta-large-openai-detector''': 5_1_2,
}


@lru_cache()
def 				_a       (				)->  List[str]:
  SCREAMING_SNAKE_CASE_							=       (
      list(range(ord('!'   )						,	ord('~'   ) + 1   )   ) + list(range(ord('¡'   )						,	ord('¬'   ) + 1   )   ) + list(range(ord('®'   )						,	ord('ÿ'   ) + 1   )   )
  )
  SCREAMING_SNAKE_CASE_							=       bs[:]
  SCREAMING_SNAKE_CASE_							=       0
  for b in range(2**8   ):
    if b not in bs:
      bs.append(lowerCAmelCase   )
      cs.append(2**8 + n   )
      n += 1
  SCREAMING_SNAKE_CASE_							=       [chr(lowerCAmelCase   ) for n in cs]
  return dict(zip(lowerCAmelCase						,	lowerCAmelCase   )   )


def 				_a       (				lowerCAmelCase   )->  Dict:
  SCREAMING_SNAKE_CASE_							=       set()
  SCREAMING_SNAKE_CASE_							=       word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)   )
    SCREAMING_SNAKE_CASE_							=       char
  return pairs




class       lowercase_		(SCREAMING_SNAKE_CASE__   ):
      lowerCAmelCase__ 			=VOCAB_FILES_NAMES
      lowerCAmelCase__ 			=PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__ 			=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__ 			=["input_ids", "attention_mask"]



      def __init__( self  :      Any				,       snake_case__  :      Dict				,       snake_case__  :      Optional[Any]				,       snake_case__  :      Optional[int]="replace"				,       snake_case__  :      List[Any]="<s>"				,       snake_case__  :      Union[str, Any]="</s>"				,       snake_case__  :      Any="</s>"				,       snake_case__  :      Any="<s>"				,       snake_case__  :      Any="<unk>"				,       snake_case__  :      Optional[int]="<pad>"				,       snake_case__  :      Any="<mask>"				,       snake_case__  :      List[str]=False				,       **snake_case__  :      List[Any]				,       ):

        """simple docstring"""

        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else bos_token
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else eos_token
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else sep_token
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else cls_token
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else unk_token
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        SCREAMING_SNAKE_CASE_							=       AddedToken(snake_case__				,       lstrip=snake_case__				,       rstrip=snake_case__     ) if isinstance(snake_case__				,       snake_case__     ) else mask_token

        super().__init__(
            errors=snake_case__				,       bos_token=snake_case__				,       eos_token=snake_case__				,       unk_token=snake_case__				,       sep_token=snake_case__				,       cls_token=snake_case__				,       pad_token=snake_case__				,       mask_token=snake_case__				,       add_prefix_space=snake_case__				,       **snake_case__				,       )

        with open(snake_case__				,       encoding='utf-8'     ) as vocab_handle:
          SCREAMING_SNAKE_CASE_							=       json.load(snake_case__     )
        SCREAMING_SNAKE_CASE_							=       {v: k for k, v in self.encoder.items()}
        SCREAMING_SNAKE_CASE_							=       errors  # how to handle errors in decoding
        SCREAMING_SNAKE_CASE_							=       bytes_to_unicode()
        SCREAMING_SNAKE_CASE_							=       {v: k for k, v in self.byte_encoder.items()}
        with open(snake_case__				,       encoding='utf-8'     ) as merges_handle:
          SCREAMING_SNAKE_CASE_							=       merges_handle.read().split('\n'     )[1:-1]
        SCREAMING_SNAKE_CASE_							=       [tuple(merge.split()     ) for merge in bpe_merges]
        SCREAMING_SNAKE_CASE_							=       dict(zip(snake_case__				,       range(len(snake_case__     )     )     )     )
        SCREAMING_SNAKE_CASE_							=       {}
        SCREAMING_SNAKE_CASE_							=       add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        SCREAMING_SNAKE_CASE_							=       re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )



      @property
      def   __a				( self  :      str     ):

        """simple docstring"""

        return len(self.encoder     )



      def   __a				( self  :      Dict     ):

        """simple docstring"""

        return dict(self.encoder				,       **self.added_tokens_encoder     )



      def   __a				( self  :      Union[str, Any]				,       snake_case__  :      List[str]     ):

        """simple docstring"""

        if token in self.cache:
          return self.cache[token]
        SCREAMING_SNAKE_CASE_							=       tuple(snake_case__     )
        SCREAMING_SNAKE_CASE_							=       get_pairs(snake_case__     )

        if not pairs:
          return token

        while True:
          SCREAMING_SNAKE_CASE_							=       min(snake_case__				,       key=lambda snake_case__     : self.bpe_ranks.get(snake_case__				,       float('inf'     )     )     )
          if bigram not in self.bpe_ranks:
            break
          SCREAMING_SNAKE_CASE_			,							SCREAMING_SNAKE_CASE_							=       bigram
          SCREAMING_SNAKE_CASE_							=       []
          SCREAMING_SNAKE_CASE_							=       0
          while i < len(snake_case__     ):
            try:
              SCREAMING_SNAKE_CASE_							=       word.index(snake_case__				,       snake_case__     )
            except ValueError:
              new_word.extend(word[i:]     )
              break
            else:
              new_word.extend(word[i:j]     )
              SCREAMING_SNAKE_CASE_							=       j

            if word[i] == first and i < len(snake_case__     ) - 1 and word[i + 1] == second:
              new_word.append(first + second     )
              i += 2
            else:
              new_word.append(word[i]     )
              i += 1
          SCREAMING_SNAKE_CASE_							=       tuple(snake_case__     )
          SCREAMING_SNAKE_CASE_							=       new_word
          if len(snake_case__     ) == 1:
            break
          else:
            SCREAMING_SNAKE_CASE_							=       get_pairs(snake_case__     )
        SCREAMING_SNAKE_CASE_							=       ' '.join(snake_case__     )
        SCREAMING_SNAKE_CASE_							=       word
        return word



      def   __a				( self  :      Optional[Any]				,       snake_case__  :      List[str]     ):

        """simple docstring"""

        SCREAMING_SNAKE_CASE_							=       []
        for token in re.findall(self.pat				,       snake_case__     ):
          SCREAMING_SNAKE_CASE_							=       ''.join(
              self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__     ).split(' '     )     )
        return bpe_tokens



      def   __a				( self  :      str				,       snake_case__  :      Dict     ):

        """simple docstring"""

        return self.encoder.get(snake_case__				,       self.encoder.get(self.unk_token     )     )



      def   __a				( self  :      Dict				,       snake_case__  :      int     ):

        """simple docstring"""

        return self.decoder.get(snake_case__     )



      def   __a				( self  :      List[Any]				,       snake_case__  :      List[Any]     ):

        """simple docstring"""

        SCREAMING_SNAKE_CASE_							=       ''.join(snake_case__     )
        SCREAMING_SNAKE_CASE_							=       bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'				,       errors=self.errors     )
        return text



      def   __a				( self  :      Optional[Any]				,       snake_case__  :      str				,       snake_case__  :      Optional[str] = None     ):

        """simple docstring"""

        if not os.path.isdir(snake_case__     ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
          return
        SCREAMING_SNAKE_CASE_							=       os.path.join(
            snake_case__				,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
        SCREAMING_SNAKE_CASE_							=       os.path.join(
            snake_case__				,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

        with open(snake_case__				,       'w'				,       encoding='utf-8'     ) as f:
          f.write(json.dumps(self.encoder				,       indent=2				,       sort_keys=snake_case__				,       ensure_ascii=snake_case__     ) + '\n'     )

        SCREAMING_SNAKE_CASE_							=       0
        with open(snake_case__				,       'w'				,       encoding='utf-8'     ) as writer:
          writer.write('#version: 0.2\n'     )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,       key=lambda snake_case__     : kv[1]     ):
            if index != token_index:
              logger.warning(
                  f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                  ' Please check that the tokenizer is not corrupted!'     )
              SCREAMING_SNAKE_CASE_							=       token_index
            writer.write(' '.join(snake_case__     ) + '\n'     )
            index += 1

        return vocab_file, merge_file



      def   __a				( self  :      Dict				,       snake_case__  :      List[int]				,       snake_case__  :      Optional[List[int]] = None     ):

        """simple docstring"""

        if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        SCREAMING_SNAKE_CASE_							=       [self.cls_token_id]
        SCREAMING_SNAKE_CASE_							=       [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep



      def   __a				( self  :      Optional[Any]				,       snake_case__  :      List[int]				,       snake_case__  :      Optional[List[int]] = None				,       snake_case__  :      bool = False     ):

        """simple docstring"""

        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=snake_case__				,       token_ids_a=snake_case__				,       already_has_special_tokens=snake_case__     )

        if token_ids_a is None:
          return [1] + ([0] * len(snake_case__     )) + [1]
        return [1] + ([0] * len(snake_case__     )) + [1, 1] + ([0] * len(snake_case__     )) + [1]



      def   __a				( self  :      List[Any]				,       snake_case__  :      List[int]				,       snake_case__  :      Optional[List[int]] = None     ):

        """simple docstring"""

        SCREAMING_SNAKE_CASE_							=       [self.sep_token_id]
        SCREAMING_SNAKE_CASE_							=       [self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep     ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]






      def   __a				( self  :      int				,       snake_case__  :      Optional[Any]				,       snake_case__  :      Any=False				,       **snake_case__  :      Dict     ):

        """simple docstring"""

        SCREAMING_SNAKE_CASE_							=       kwargs.pop('add_prefix_space'				,       self.add_prefix_space     )
        if (is_split_into_words or add_prefix_space) and (len(snake_case__     ) > 0 and not text[0].isspace()):
          SCREAMING_SNAKE_CASE_							=       ' ' + text
        return (text, kwargs)





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


SCREAMING_SNAKE_CASE:    List[str]											=   '''
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
'''



class       lowercase_		(unittest.TestCase       ,  SCREAMING_SNAKE_CASE__   ):



						def   __a				( self  :      Dict     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       load_tool('summarization'     )
								self.tool.setup()
								SCREAMING_SNAKE_CASE_							=       load_tool('summarization'				,       remote=snake_case__     )



						def   __a				( self  :      Union[str, Any]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       self.tool(snake_case__     )
								self.assertEqual(
								    snake_case__				,       'Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM.'				,       )



						def   __a				( self  :      List[str]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       self.remote_tool(snake_case__     )
								self.assertEqual(
								    snake_case__				,       'Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM.'				,       )



						def   __a				( self  :      Optional[Any]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       self.tool(text=snake_case__     )
								self.assertEqual(
								    snake_case__				,       'Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM.'				,       )






						def   __a				( self  :      List[Any]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       self.remote_tool(text=snake_case__     )
								self.assertEqual(
								    snake_case__				,       'Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM.'				,       )